diff --git a/.bazelignore b/.bazelignore index 1ef6fd4c..e0df23f9 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,3 +1,4 @@ .* -third_party/modules .zig-cache +.devenv +venv/ diff --git a/.bazelrc b/.bazelrc index e386fc58..09c50737 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,21 +1,15 @@ # Faster hash function startup --digest_function=blake3 -# Disable WORKSPACE -common --enable_workspace=false - -# Don't need for now -common --lockfile_mode=update - # Needed by LLVM and/or XLA common --experimental_repo_remote_exec -common --experimental_remote_cache_eviction_retries=5 # Self explanatory common --enable_platform_specific_config # Ensure host tools are built optimized common --host_compilation_mode=opt +common --@rules_zig//zig/settings:host_mode=release_safe # Suppress all warnings, LLVM and friends are too noisy common --copt='-w' @@ -26,35 +20,33 @@ common --cxxopt='-std=c++17' common --host_cxxopt='-std=c++17' common --enable_runfiles -# Ensure the local env doesn't leak into the build -common --incompatible_strict_action_env +# Ensure rules_python doesn't rely on the system's Python for stage-1 bootstrap +common --@rules_python//python/config_settings:bootstrap_impl=script # We don't to check the built files in the sandbox or repositories common --experimental_check_output_files=false -# Performance -common --legacy_external_runfiles=false - -common --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline - -# Mount the zig cache into the sandboxes -build:macos --sandbox_add_mount_pair=/var/tmp +# Expose Zig's shared cache directories to sandboxed actions when the +# selected sandbox implementation supports host path mounts. +# +# These flags are consumed by linux-sandbox, and by Bazel's Docker sandbox. +# They are ignored by processwrapper-sandbox and macOS sandbox-exec, which use +# a symlinked execroot rather than bind mounts. build:linux --sandbox_add_mount_pair=/tmp +build:linux --sandbox_writable_path=/tmp +build:linux --sandbox_add_mount_pair=/var/tmp +build:linux --sandbox_writable_path=/var/tmp # Ensure workers are sandboxed build --worker_sandboxing -# Allows symlinks with spaces -build --experimental_inprocess_symlink_creation - # Build in debug for now, by default common --compilation_mode=dbg # Various cc options -common --experimental_cc_static_library common --experimental_cc_shared_library common --incompatible_use_cc_configure_from_rules_cc -common --experimental_starlark_cc_import +common --incompatible_autoload_externally=+@rules_cc,+@rules_python,+@rules_shell,+@rules_java common --experimental_platform_in_output_dir @@ -64,28 +56,33 @@ build --experimental_inmemory_sandbox_stashes # clang by default common --compiler=clang -# Enable resolving proto compiler from toolchains -common --incompatible_enable_proto_toolchain_resolution - # Allow empty globs since some legacy rules common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code -common --@rules_zig//zig/settings:use_cc_common_link=True -# Zig self hosted backend is too fragile for now -common --@rules_zig//zig/settings:zigopt=-fllvm +common --@rules_zig//zig/settings:use_cc_common_link=true -common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True +common --@rules_zig//zig/settings:use_standalone_translate_c=True + +common --@llvm//config:experimental_stub_libgcc_s=true common --macos_minimum_os=13.0 +# Enable stamping +common --stamp=true +common --workspace_status_command=./tools/workspace_status.sh + common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 +common --xcode_version_config=//:disable_xcode # Required for protobuf upb on Linux common:linux --copt=-fPIC +# Prefer prebuilt protoc (finally!) +common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc=true + # Print test errors in the console test --test_output=errors @@ -108,9 +105,43 @@ common:alldebug --strategy=local common:release --compilation_mode=opt common:release --copt=-Ofast +common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe +common:release --@rules_zig//zig/settings:zigopt=-fllvm # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native -common:native --@rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native + +# Remote execution and caching with BuildBuddy +common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ +common:remote --bes_backend=grpcs://remote.buildbuddy.io +common:remote --remote_cache=grpcs://remote.buildbuddy.io +common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io +common:remote --remote_executor=grpcs://remote.buildbuddy.io +common:remote --remote_download_toplevel +common:remote --remote_download_outputs=minimal +common:remote --nobuild_runfile_links +common:remote --remote_timeout=3600 +common:remote --experimental_remote_cache_chunking +common:remote --noexperimental_throttle_remote_action_building +common:remote --remote_cache_compression +# Re-runs actions instead of failing on transient lost remote/disk cache inputs. +common:remote --rewind_lost_inputs +# Falls back to local downloads if the remote asset downloader is unavailable. +common:remote --experimental_remote_downloader_local_fallback +common:remote --grpc_keepalive_time=30s +common:remote --jobs=2000 +common:remote --extra_execution_platforms=@zml//:rbe_platform +common:remote --experimental_repo_remote_exec=true + +# Runtime profiling helpers for `bazel run`. +run:nsys --@zml//platforms:cuda=true +run:nsys --run_under=//tools:nsys + +run:rocprofv3 --@zml//platforms:rocm=true +run:rocprofv3 --run_under=//tools:rocprofv3 + +run:neuron-profile --@zml//platforms:neuron=true +run:neuron-profile --@zml//platforms:cpu=false +run:neuron-profile --run_under='//tools/neuron:profile /tmp/zml-neuron-profile' diff --git a/.bazelversion b/.bazelversion index f9c71a52..b13d146a 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.5.1 +9.3.0 diff --git a/.github/ISSUE_TEMPLATE/llmd--bug-report.md b/.github/ISSUE_TEMPLATE/llmd--bug-report.md new file mode 100644 index 00000000..b9629993 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/llmd--bug-report.md @@ -0,0 +1,34 @@ +--- +name: 'LLMD: Bug report' +about: If you hit a bug in LLMD, let us know +title: "[LLMD]" +labels: LLMD +assignees: '' +type: Bug + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Configuration:** + - Model: [e.g. `google/gemma-4-31B-it`] + - OS: [e.g. `Ubuntu 24.04`] + - Platform: [`cuda` / `rocm` / `oneapi` / `metal` / `tpu`] + - Accelerator: [e.g. `AMD MI300X`] + - Driver version: [e.g. `595.71.05`] + - Version: [e.g. 0.1.2 or docker image `sha256@`] + - Command line: [e.g. `docker run ...`] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 09d88bf4..aefeccc9 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,10 +3,30 @@ images: platform: "linux" arch: "amd64" ami: "ami-04a92520784b93e73" - disk: large - preinstall: | - #!/bin/bash - wget https://github.com/jbdalido/bazel-remote/releases/download/test/bazel-remote -O /usr/local/bin/bzlcache - chmod +x /usr/local/bin/bzlcache - echo "Run bazel cache" - systemd-run --property=Restart=always /usr/local/bin/bzlcache --max_size=1000 --grpc_address 127.0.0.1:15501 --dir=/tmp/cache --s3.region eu-west-3 --s3.bucket runs-on-s3bucketcache-jycgicewbssh --s3.auth_method iam_role --s3.prefix cache --s3.endpoint=s3.eu-west-3.amazonaws.com --storage_mode uncompressed + + ubuntu24-arm64: + platform: "linux" + arch: "arm64" + ami: "ami-035c36ba717c3d79f" + +runners: + large-runner-amd64: + cpu: [16] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] + + large-runner-arm64: + cpu: [16] + family: ["c8g", "c7g"] + image: ubuntu24-arm64 + volume: 80gb + extras: ["s3-cache"] + + small-runner-amd64: + cpu: [4] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48bc9320..31bc46e8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,98 +12,174 @@ concurrency: cancel-in-progress: ${{ !contains(github.ref, 'master')}} jobs: - platforms: + ci: + name: ci-${{ matrix.target }}-${{ matrix.arch }} strategy: + fail-fast: false matrix: - runtimes: - - target: cuda - bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" - enable_cache: false - user: actions - runs-on: ["self-hosted", "platform=cuda"] - - target: rocm - bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" - enable_cache: false - user: actions - runs-on: ["self-hosted", "platform=rocm"] + include: - target: cpu - bazel_opts: "--@zml//platforms:cpu=true" - enable_cache: true - user: runner - runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] - runs-on: ${{ matrix.runtimes.runs-on }} + arch: amd64 + bazel_common_opts: "" + run_tests: true + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu-amd64 + - runner=large-runner-amd64 + + - target: neuron + arch: amd64 + bazel_common_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron-amd64 + - runner=large-runner-amd64 + + - target: tpu + arch: amd64 + bazel_common_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu-amd64 + - runner=large-runner-amd64 + + - target: cuda + arch: amd64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + run_tests: true + runner_labels: ["self-hosted", "linux", "x64", "platform=cuda"] + + - target: cuda + arch: arm64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cuda-arm64 + - runner=large-runner-arm64 + + - target: rocm + arch: amd64 + bazel_common_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" + run_tests: true + runner_labels: ["self-hosted", "linux", "x64", "platform=rocm"] + + - target: oneapi + arch: amd64 + bazel_common_opts: "--@zml//platforms:oneapi=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-oneapi-amd64 + - runner=large-runner-amd64 + + - target: metal + arch: arm64 + bazel_common_opts: "--@zml//platforms:metal=true --@zml//platforms:cpu=false" + bazel_build_opts: "--platforms=@zml//platforms:macos_arm64" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-metal-arm64 + - runner=large-runner-amd64 + + - target: furiosa + arch: amd64 + bazel_common_opts: "--@zml//platforms:furiosa=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-furiosa-amd64 + - runner=large-runner-amd64 + + runs-on: ${{ matrix.runner_labels }} steps: - - uses: runs-on/action@v1 + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - name: Checkout - uses: actions/checkout@v4 - - - uses: runs-on/snapshot@v1 - if: ${{ matrix.runtimes.enable_cache }} - with: - path: /home/${{ matrix.runtimes.user }}/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - if: ${{ matrix.runtimes.enable_cache }} - run: sudo chown -R ${{ matrix.runtimes.user }}:${{ matrix.runtimes.user }} /home/${{ matrix.runtimes.user }}/.cache/ + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Ensure self hosted cache permissions - if: ${{ !matrix.runtimes.enable_cache }} + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - - uses: bazel-contrib/setup-bazel@0.15.0 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed + bazelisk-version: 1.29.0 + bazelisk-cache: true + cache-save: false bazelrc: | - common -c opt + startup --experimental_remote_repo_contents_cache common --verbose_failures common --keep_going common --test_output=streamed common --color=yes common --show_timestamps - common --disk_cache=/home/${{ matrix.runtimes.user }}/.cache/bazel-disk - common --strip=always + common --config=remote + common --strategy=TestRunner=local + ${{ format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} - - name: ZML / bazel build //... - run: bazel build ${{ matrix.runtimes.bazel_opts }} //... + - name: Capture self hosted storage usage before + id: storage_before + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: &measure-self-hosted-storage | + output_base=$(bazel info output_base) || output_base="" + bazel_size="unavailable" + if [[ -n "$output_base" ]]; then + bazel_size=$(du -sh --si -- "$output_base" | cut -f1) || bazel_size="unavailable" + fi + disks=$(df -H --local --output=source,size,used,avail,pcent,target | + awk '$1 ~ /^\/dev\// && $1 !~ /^\/dev\/loop/ && !seen[$1]++ { + printf "%s%s: %s / %s used (%s), %s free", separator, $6, $3, $2, $5, $4 + separator = "; " + } END {if (separator == "") printf "none"}') || disks="unavailable" + { + printf 'output_base=%s\n' "$output_base" + printf 'bazel_size=%s\n' "$bazel_size" + printf 'disks=%s\n' "$disks" + } >> "$GITHUB_OUTPUT" - - name: ZML / bazel test //... - run: bazel test ${{ matrix.runtimes.bazel_opts }} //... - ci: + - run: bazel mod graph + - run: bazel query ${{ matrix.bazel_common_opts }} --output=graph //... + - run: bazel cquery ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + - run: bazel build ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + - if: ${{ matrix.run_tests }} + run: bazel test ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + + - name: Capture self hosted storage usage after + id: storage_after + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: *measure-self-hosted-storage + + - name: Report self hosted storage usage + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + env: + STORAGE_JOB: ${{ github.job }}-${{ matrix.target }}-${{ matrix.arch }} + OUTPUT_BASE: ${{ steps.storage_after.outputs.output_base || steps.storage_before.outputs.output_base || 'unavailable' }} + SIZE_BEFORE: ${{ steps.storage_before.outputs.bazel_size || 'unavailable' }} + SIZE_AFTER: ${{ steps.storage_after.outputs.bazel_size || 'unavailable' }} + DISKS_BEFORE: ${{ steps.storage_before.outputs.disks || 'unavailable' }} + DISKS_AFTER: ${{ steps.storage_after.outputs.disks || 'unavailable' }} + run: | + message=$(printf 'Runner: %s\nBazel output base: %s\nBazel directory size: %s -> %s\nDisks before: %s\nDisks after: %s' \ + "$RUNNER_NAME" "$OUTPUT_BASE" "$SIZE_BEFORE" "$SIZE_AFTER" "$DISKS_BEFORE" "$DISKS_AFTER") + message=${message//'%'/'%25'} + message=${message//$'\r'/'%0D'} + message=${message//$'\n'/'%0A'} + printf '::notice title=Disk usage - %s::%s\n' "$STORAGE_JOB" "$message" + + lint: runs-on: - [ - "runs-on", - "runner=4cpu-linux-x64", - "image=ubuntu24-amd64", - "run-id=${{ github.run_id }}", - "extras=s3-cache", - ] + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-lint + - runner=small-runner-amd64 steps: - - uses: runs-on/action@v1 + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - name: Checkout - uses: actions/checkout@v4 - - - uses: runs-on/snapshot@v1 - with: - path: /home/runner/.cache/bazel-disk - volume_size: 350 - - - name: TMP fix for the mount user - run: sudo chown -R runner:runner /home/runner/.cache/ + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: - version: 0.15.2 + version: 0.17.0 mirror: https://mirror.zml.ai/zig - name: Zig Format @@ -111,27 +187,3 @@ jobs: - name: No autofix in Zig code run: bash -c "! git grep -l '// autofix' -- '*.zig'" - - - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-version: 1.26.0 - bazelisk-cache: false - # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed - bazelrc: | - common -c opt - common --verbose_failures - common --keep_going - common --test_output=streamed - common --color=yes - common --show_timestamps - common --disk_cache=/home/runner/.cache/bazel-disk - common --strip=always - - - name: ZML / bazel mod explain - run: bazel mod explain - - - name: ZML / bazel mod graph - run: bazel mod graph - - - name: ZML / bazel query //... - run: bazel query //... diff --git a/.gitignore b/.gitignore index a4b52b95..0ec34365 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,14 @@ venv # Editor specific ## Neovim .nvim.local.lua + +# Devenv +.devenv* +devenv.local.nix +devenv.local.yaml + +# direnv +.direnv + +# pre-commit +.pre-commit-config.yaml diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..529cdcf3 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,19 @@ +[[language]] +auto-format = true +file-types = [ "zig" ] +language-servers = [ "zls" ] +name = "zig" +roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] +scope = "source.zig" + +[language.indent] +tab-width = 4 +unit = " " + +[language-server.zls] +command = "sh" +args = [ + "-c", + "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", + "zls" +] diff --git a/.nvim.lua b/.nvim.lua index a1f9ad95..9982de80 100644 --- a/.nvim.lua +++ b/.nvim.lua @@ -4,33 +4,63 @@ local cwd = vim.fn.getcwd() -- Load the local config file if it exists local f, err = loadfile(".nvim.local.lua") if not err then - f() + local ok, err_f = pcall(f) + if not ok then + vim.print(err_f) + end else - local capabilities = vim.lsp.protocol.make_client_capabilities() + vim.lsp.config("zls", { + cmd = { cwd .. "/tools/zls.sh" }, + }) + vim.lsp.enable("zls") - -- Prepend CWD to relative paths - local zls_cmd = cwd .. "/../zml/tools/zls.sh" + function runFixAll() + local params = vim.lsp.util.make_range_params() + params.context = { + only = { "source.fixAll" }, + diagnostics = {}, + } - vim.lsp.config["zls"] = { - capabilities = capabilities, - cmd = { zls_cmd }, - root_marker = { "build.zig" }, - filetypes = {"zig"}, - } - vim.lsp.enable('zls') + -- Send synchronous request (timeout 2 seconds) + local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) + if not responses then + return + end - vim.api.nvim_create_autocmd('BufWritePre',{ - pattern = {"*.zig", "*.zon"}, - callback = function(ev) - vim.lsp.buf.code_action({ - context = { only = { "source.organizeImports" } }, - apply = true, - }) - vim.lsp.buf.code_action({ - context = { only = { "source.fixAll" } }, - apply = true, - }) - vim.loop.sleep(100) - end - }) + for _, resp in pairs(responses) do + for _, action in ipairs(resp.result or {}) do + -- If action has edits, apply them + if action.edit then + vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") + end + -- If the action has a command, execute it + if action.command then + vim.lsp.buf.execute_command(action.command) + end + end + end + end + + vim.api.nvim_create_autocmd("BufWritePre", { + pattern = { "*.zig", "*.zon" }, + callback = runFixAll, + }) + + local ok, conform = pcall(require, "conform") + + if ok then + conform.formatters_by_ft.bzl = { "buildifier" } + conform.formatters.buildifier = { + command = cwd .. "/tools/buildifier.sh", + } + + vim.api.nvim_create_autocmd("FileType", { + pattern = "bzl", + callback = function() + vim.keymap.set("n", "f", function() + conform.format() + end, { remap = false, silent = true }) + end, + }) + end end diff --git a/.vscode/settings.json b/.vscode/settings.json index 82050af6..772705fb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,15 +9,13 @@ "**/bazel-*": true }, "files.watcherExclude": { - "**/bazel-*": true, + "**/bazel-*": true }, // "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "bazel.buildifierFixOnFormat": true, "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": [ - "." - ], + "bazel.pathsToIgnore": ["."], "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", "C_Cpp.formatting": "clangFormat", "C_Cpp.clang_format_style": "Microsoft", @@ -28,10 +26,7 @@ "editor.suggest.insertMode": "replace", "editor.stickyScroll.defaultModel": "foldingProviderModel", "editor.codeActionsOnSave": { - "source.fixAll": "explicit", + "source.fixAll": "explicit" } - }, - "rust-analyzer.linkedProjects": [ - "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" - ] + } } diff --git a/.zed/settings.json b/.zed/settings.json index 141b19f4..55eff7c8 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -8,12 +8,12 @@ "formatter": "language_server", "code_actions_on_format": { "source.fixAll": true, - "source.organizeImports": true, - }, + "source.organizeImports": true + } }, "YAML": { - "tab_size": 2, - }, + "tab_size": 2 + } }, "file_scan_exclusions": [ "**/.git", @@ -25,17 +25,17 @@ "**/Thumbs.db", "**/.classpath", "**/.settings", - "**/bazel-*", + "**/bazel-*" ], "tab_size": 4, "inlay_hints": { - "enabled": true, + "enabled": true }, "lsp": { "zls": { "binary": { - "path": "tools/zls.sh", - }, - }, - }, + "path": "tools/zls.sh" + } + } + } } diff --git a/AGENTS.md b/AGENTS.md index 834e379e..fe0f5d9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,157 +1,35 @@ -# AGENTS.md - ZML Agent Guide +# Repository Guidelines -This guide is for autonomous coding agents working in this repository. +## Project Structure & Module Organization -## Scope and Environment +ZML is a Zig project built with Bazel. Core library code lives in `zml/`, with IO under `zml/io/`, tokenizers under `zml/tokenizer/`, attention implementations under `zml/attention/`, and MoE support under `zml/moe/`. Shared Zig utilities are in `stdx/`; FFI and C helpers are in `ffi/` and `upb/`. Bazel rules and workspace helpers live in `bazel/` and `third_party/`. Runnable examples are in `examples/`, while user-facing documentation is in `docs/`. Keep tests close to implementation files unless a package already has a separate test file. -- Primary language: Zig (dev snapshot), built and tested with Bazel. -- Main subsystems: `zml/`, `stdx/`, `mlir/`, `pjrt/`, `platforms/`, `examples/`. -- Bazel version is pinned in `.bazelversion` (`8.5.1`). -- Build mode defaults to debug via `.bazelrc` (`--compilation_mode=dbg`). +## Build, Test, and Development Commands -## External Instruction Files +- `bazel build //zml`: builds the main ZML library. +- `bazel test //zml:test`: runs the core Zig test target. +- `bazel test //stdx:test`: runs shared utility tests. +- `bazel test //zml/tokenizer:test`: runs tokenizer tests. +- `bazel run //examples/mnist`: runs the quick MNIST smoke test. +- `bazel run //examples/llm -- --model=hf://... --prompt="..."`: runs the LLM example. +- `bazel build //...` and `bazel test //...`: use before broad or shared changes. +- `./tools/buildifier.sh`: formats Bazel/Starlark files. +- `echo "$(bazel info output_base)/$(bazel cquery --output=files "$(bazel cquery "filter('zig_toolchain', deps(//zml))" 2>/dev/null | cut -d' ' -f1 | head -n1)" 2>/dev/null | rg "/lib\$")/std"`: find the current Zig standard library used by Bazel. -- Cursor rules: `.cursor/rules/**` and `.cursorrules` are not present. -- Copilot rules: `.github/copilot-instructions.md` is not present. -- If any of these files appear later, treat them as required constraints. +Append platform flags when relevant, for example `--@zml//platforms:cuda=true`, `--@zml//platforms:rocm=true`, `--@zml//platforms:tpu=true`, or `--@zml//platforms:cpu=false`. -## Core Build Commands +## Coding Style & Naming Conventions -```bash -# Build all targets -bazel build //... +Follow the Zig style guide and format Zig code with `zig fmt`; CI checks `zig fmt --check` outside `third_party/`. Use four-space indentation. Prefer `const x: Foo = .{ .bar = 1 };`, `pub fn method(self: Foo)`, and module imports such as `const foo = @import("foo.zig"); foo.bar()`. Use `PascalCase` for types and `lowerCamelCase` for functions, fields, and locals. Add comments only for non-obvious invariants, ownership, tensor shape semantics, or platform behavior. -# Build specific targets/packages -bazel build //zml -bazel build //examples/llama -bazel build //examples/mnist +## Testing Guidelines -# Alternate build configs -bazel build --config=debug //... -bazel build --config=alldebug //... -bazel build --config=release //... -bazel build --config=native //... -``` +Tests are generally inline Zig `test "..." {}` blocks near the code they cover. Use descriptive test names and narrow Bazel targets first, then broaden when touching shared APIs. For tensor or graph changes, cover shape/tag inference and runtime behavior when practical. CI builds all targets and runs tests on CPU, CUDA, and ROCm where available. -### Useful `.bazelrc` config meanings +## Commit & Pull Request Guidelines -- `--config=debug`: optimized backend, Zig frontend debug mode. -- `--config=release`: optimized (`-Ofast`, Zig `release_safe`). -- `--config=alldebug` / `--config=native` / `--config=silent`: debug-all, local CPU tuning, reduced UI noise. +Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. -## Test Commands +## Quirks -```bash -# Run everything -bazel test //... - -# Common package tests -bazel test //zml:test -bazel test //stdx:test -bazel test //mlir:test -bazel test //zml/tokenizer:test - -# Discover Zig test targets -bazel query 'kind(zig_test, //...)' -``` - -### Run a single test (inside a `zig_test` target) - -`zml/test_runner.zig` supports substring filtering from positional args. -Pass filter with `--test_arg`: - -```bash -# Runs tests whose name contains "normalizeL2" -bazel test //zml:test --test_arg=normalizeL2 - -# Example on another target -bazel test //stdx:test --test_arg=json -``` - -- Matching is substring-based on test function identifier; `.bazelrc` already sets `test --test_output=errors`. - -## Lint / Formatting Commands - -```bash -# Format Bazel/Starlark files -./tools/buildifier.sh - -# Run ZLS through Bazel (editor and language tooling) -./tools/zls.sh -``` - -## Platform / Accelerator Flags - -Append these to `bazel build|test|run` as needed: - -```bash ---@zml//platforms:cuda=true ---@zml//platforms:rocm=true ---@zml//platforms:tpu=true ---@zml//platforms:neuron=true ---@zml//platforms:cpu=false -``` - -## Code Style (Repository Conventions) - -### Imports and namespacing - -- Prefer module imports, then namespaced access. -- Prefer: `const foo = @import("foo.zig"); foo.bar()`. -- Avoid: `const bar = @import("foo.zig").bar` as default style. -- Import specific types directly only when heavily used. -- Keep local imports relative within a package. - -### Formatting and structure - -- Follow Zig style and existing local patterns first. -- Prefer typed anonymous initialization where clear: `const x: Foo = .{ .bar = 1 };`. -- Keep functions focused; pair resource acquisition with nearby `defer`/`errdefer`. -- Add comments only for non-obvious invariants, ownership, or shape semantics. - -### Types, naming, and tensor semantics - -- Types/structs: `PascalCase`. -- Functions/vars/fields: Zig lower camel case. -- Keep names descriptive; avoid short opaque identifiers unless very local. -- Use semantic tensor tags (`.b`, `.s`, `.d`, `.h`, `.hd`, etc.) over raw index assumptions. - -### Errors, assertions, and cleanup - -- Prefer `try`-based propagation. -- Use `catch |err|` when you add useful recovery or context. -- Use `errdefer` on partial initialization paths. -- Prefer `stdx.debug.assert` over bare `std.debug.assert` in this codebase. -- Avoid panic-style control flow except for unrecoverable programmer errors. - -### Logging and CLI entrypoints - -- Use scoped logs (for example `std.log.scoped(.@"zml/io")`). -- CLI binaries commonly define `pub const std_options`. -- Argument parsing usually follows `stdx.flags` conventions. - -## Testing Patterns - -- Tests are usually inline Zig `test "..." {}` blocks near implementation. -- Common helpers: `zml.testing.env`, `zml.testing.expectClose`, `zml.testing.approxEq`. -- `zig_test` targets often use `zml/test_runner.zig` for progress/filter/leak checks. - -## Critical Architecture Guardrails - -- `Tensor` ops in `forward` build an MLIR graph; they do not execute immediately. -- Preserve distinctions: - - `Shape` = metadata only - - `Tensor` = compile-time graph value - - `Buffer` = runtime device allocation - - `Slice` = host-side data view -- `zml.Bufferized(T)` maps tensor-typed model structures to runtime buffer structures. -- Weight loading paths are centered on safetensors + TensorStore/VFS. - -## Practical Agent Workflow - -- Read the target file, nearby tests, and owning `BUILD.bazel`. -- Check whether behavior lives in compile-time graph construction or runtime execution. - -- Run the narrowest relevant tests first (`--test_arg` when possible). -- Run broader package tests if shared code changed. -- Run `./tools/buildifier.sh` when any Bazel files changed. +If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/BUILD.bazel b/BUILD.bazel index b3dd9f68..734a3792 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,15 +1,48 @@ -load("//third_party/zls:zls_completion.bzl", "zls_completion") +load("@@apple_support+//xcode:xcode_config.bzl", "xcode_config") +load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") +load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") + +# Dummy xcode_config so we don't need to build xcode_locator in the repository rule. +xcode_config( + name = "disable_xcode", + visibility = ["//visibility:public"], +) + +alias( + name = "translate-c", + actual = "@translate-c//:translate-c", + visibility = ["//visibility:public"], +) + +translate_c_toolchain( + name = "translate_c_toolchain_impl", + runtime_modules = [ + "@translate-c//:c_builtins", + "@translate-c//:helpers", + ], + translate_c = "@translate-c//:translate-c", +) + +toolchain( + name = "translate_c_toolchain", + target_settings = ["@rules_zig//zig/config/use_standalone_translate_c:enabled"], + toolchain = ":translate_c_toolchain_impl", + toolchain_type = "@rules_zig//zig/translate-c:toolchain_type", + visibility = ["//visibility:public"], +) zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//examples/lfm", - "//examples/transfer-benchmark", - "//examples/llama", + "//bin/zml-smi", + "//examples/llm", "//examples/mnist", + "//examples/sharding", + "//examples/io:playground", "//stdx", "//zml", + "//zml/tokenizer/iree", ], ) @@ -20,11 +53,11 @@ platform( "@platforms//os:linux", # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) # But apparently not in this docker://ubuntu:22.04 image? - "@toolchains_llvm_bootstrapped//constraints/libc:gnu.2.31", + "@llvm//constraints/libc:gnu.2.28", "@bazel_tools//tools/cpp:clang", ], exec_properties = { - "container-image": "docker://ubuntu:22.04", + "container-image": "docker://mirror.gcr.io/library/ubuntu:22.04", "OSFamily": "Linux", }, visibility = ["//visibility:public"], diff --git a/MODULE.bazel b/MODULE.bazel index 0faba184..e9c4c9d3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,33 +2,107 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20260107.1") -bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") +bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") + +# Work around abseil-cpp#2071 until the BCR release includes the Clang BMI2 fix. +single_version_override( + module_name = "abseil-cpp", + patch_strip = 1, + patches = [ + "//:third_party/abseil-cpp/bmi2intrin.patch", + ], + version = "20260526.0", +) + +bazel_dep(name = "bazel_lib", version = "3.7.2") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") + +## Needed by iree +bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") -bazel_dep(name = "platforms", version = "1.0.0") -bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") -bazel_dep(name = "rules_cc", version = "0.2.16") -bazel_dep(name = "rules_distroless", version = "0.6.2") +bazel_dep(name = "platforms", version = "1.1.0") +bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf") +bazel_dep(name = "pybind11_bazel", version = "3.0.1") +bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_cc", version = "0.2.19") +bazel_dep(name = "rules_distroless", version = "0.8.0") + +# Once the rules_distroless can parse unspaced apt version constraints +# Update the rules_distroless version and remove the patch +# https://github.com/bazel-contrib/rules_distroless/pull/229 +single_version_override( + module_name = "rules_distroless", + patch_strip = 1, + patches = [ + "//:third_party/rules_distroless/accept-nospace-version-constraints.patch", + "//:third_party/rules_distroless/flat-repositories.patch", + ], + version = "0.8.0", +) + bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "1.8.4") -bazel_dep(name = "rules_rust", version = "0.68.1") -bazel_dep(name = "rules_rs", version = "0.0.28") +bazel_dep(name = "rules_python", version = "1.9.0") bazel_dep(name = "rules_shell", version = "0.6.1") -bazel_dep(name = "rules_zig", version = "0.12.2") -bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") -bazel_dep(name = "with_cfg.bzl", version = "0.14.1") -bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) +# repo_name = None prevents rules_swift from eagerly auto-registering toolchains. +bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) +bazel_dep(name = "rules_zig", version = "0.17.0") + +bazel_dep(name = "tar.bzl", version = "0.10.4") +bazel_dep(name = "llvm", version = "0.8.24") +bazel_dep(name = "libxml2", version = "2.15.3") +bazel_dep(name = "xz", version = "5.4.5.bcr.9", repo_name = "llvm_xz") +bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") +bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") +bazel_dep(name = "with_cfg.bzl", version = "0.14.4") +bazel_dep(name = "highway", version = "1.4.0") +bazel_dep(name = "grpc", version = "1.80.0") +single_version_override( + module_name = "grpc", + patch_strip = 1, + patches = [ + "//:third_party/grpc/protobuf-35-upb-load.patch", + "//:third_party/grpc/grpc++_unsecure-visibility.patch", + "//:third_party/grpc/plugin-flags.patch", + ], + version = "1.80.0", +) + +single_version_override( + module_name = "envoy_api", + version = "0.0.0-20250128-4de3c74", +) + +# Added to support bazel 9+ since an old version is pulled by GRPC. +bazel_dep(name = "rules_foreign_cc", version = "0.15.1") + +bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) + +# BSDTar toolchains +tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") +use_repo( + tar_toolchains, + "bsd_tar_toolchains_darwin_amd64", + "bsd_tar_toolchains_darwin_arm64", + "bsd_tar_toolchains_linux_amd64", + "bsd_tar_toolchains_linux_arm64", + "bsd_tar_toolchains_windows_amd64", + "bsd_tar_toolchains_windows_arm64", +) +### ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.16.0-dev.2611+f996d2866") +zig.toolchain( + # Ensure Zig actions are run locally only + extra_exec_compatible_with = ["//platforms:local_execution"], + zig_version = "0.17.0", +) zig.mirrors(urls = [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -50,30 +124,54 @@ use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") + +register_toolchains("//:translate_c_toolchain") + +# Add the local execution platform to the set of eligible execution platforms +# This is used so that Zig toolchains that are only declared as compatible with +# the constraints described by this platform be selected and matched with only +# this platform only. +# This ensures Zig actions only run locally because they're slower remotely +# due to the Zig compilation model which concentrates one action for the whole +# build graph. +register_execution_platforms("//platforms:local_execution_platform") + +register_toolchains("//third_party/zls:toolchain") + ### ### LLVM Bootstrapped toolchains -register_toolchains( - "@toolchains_llvm_bootstrapped//toolchain:all", -) +register_toolchains("@llvm//toolchain:all") ### uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) -uv.configure(version = "0.6.5") +uv.configure(version = "0.9.30") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_versions") +use_repo(python, "python_3_12", "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip.parse( + +pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip_neuron.parse( download_only = True, + extra_pip_args = [ + "--abi=cp312", + "--implementation=cp", + "--python-version=312", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64", + "--platform=manylinux_2_27_x86_64", + "--platform=manylinux_2_28_x86_64", + "--platform=manylinux_2_29_x86_64", + ], hub_name = "neuron_py_deps", python_version = "3.12", requirements_lock = "//platforms/neuron:requirements.lock.txt", target_platforms = ["linux_x86_64"], ) -use_repo(pip, "neuron_py_deps") +use_repo(pip_neuron, "neuron_py_deps") # For downloading models pip.parse( @@ -84,41 +182,170 @@ pip.parse( ) use_repo(pip, "huggingface_hub") -cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") -use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") +pip.parse( + download_only = True, + # fsspec[gcs] -> gcsfs -> fsspec is a circular dependency, install them together. + experimental_requirement_cycles = { + "fsspec": [ + "fsspec", + "gcsfs", + ], + }, + hub_name = "xprof", + python_version = "3.12", + requirements_lock = "//tools/xprof:requirements.lock.txt", +) +use_repo(pip, "xprof") + +cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_plugin") +use_repo(cpu, "libzml_cpu_darwin_amd64", "libzml_cpu_darwin_arm64", "libzml_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "libpjrt_cuda") +use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libzml_cuda_linux_amd64", "libzml_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas") +use_repo( + rocm, + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libzml_rocm", +) + +oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") +use_repo(oneapi, "libzml_oneapi") + +furiosa = use_extension("//platforms/furiosa:furiosa.bzl", "furiosa_packages") +use_repo(furiosa, "libzml_furiosa") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") -use_repo(tpu, "libpjrt_tpu") +use_repo(tpu, "libzml_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libstdcpp6", "libzml_neuron") -zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") -use_repo(zls, "zls_toolchains") +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -register_toolchains("@zls_toolchains//:all") +http_archive( + name = "sevenzip_darwin_amd64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) + +# Host tools for the portable AEA extraction path. +http_archive( + name = "sevenzip_darwin_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) + +http_archive( + name = "sevenzip_linux_amd64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "4ca3b7c6f2f67866b92622818b58233dc70367be2f36b498eb0bdeaaa44b53f4", + urls = ["https://www.7-zip.org/a/7z2501-linux-x64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_amd64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "923c1caa44d3adaf2b792725c6209e879a13ab68af462ae5adfe1d611e2f3653", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_x86_64.tar.gz"], +) + +http_archive( + name = "sevenzip_linux_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "39c5140f02ce4436599303c59a149f654cb1bbc47cdc105a942120d747ae040d", + urls = ["https://www.7-zip.org/a/7z2501-linux-arm64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_arm64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "fdef33729a0cb1d6be0cbffe677a97fb167e6af07fda3eafd03137ce607e92b4", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_arm64.tar.gz"], +) + +http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") + +http_aea_archive( + name = "metal_toolchain", + archive_decryption_key = "v7VbaT39JmQPp5p4N3HoVPAKcZ5vyhY+nnjMRD79z5E=", + build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", + canonical_id = "com.apple.MobileAsset.MetalToolchain-17B54", + includes = ["Metal.xctoolchain/*"], + sha256 = "84f1cec3984878294777349e794f7bff0407e501a728bd600dc8605fa36abf6f", + urls = [ + "https://updates.cdn-apple.com/2025/mobileassets/022-20354/192ADD28-31EE-43C8-972D-E5BED5FB1845/com_apple_MobileAsset_MetalToolchain/637B0E26-E00A-488B-BA3B-CB50A8ECFD74.aar", + ], +) + +metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") +use_repo(metal, "libzml_metal") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla") +use_repo( + non_module_deps, + "arocc", + "cloud_accelerator_diagnostics", + "com_google_sentencepiece", + "cuda_tile", + "cute_ir", + "flashinfer_cutlass_moe_linux_amd64", + "flashinfer_cutlass_moe_linux_arm64", + "flydsl", + "iree", + "libvaxis", + "linenoise", + "mnist", + "org_swig_swig", + "stb", + "translate-c", + "uucode", + "xla", + "zigimg", + "zio", + "zls", + "zls_diffz", + "zls_known_folders", + "zls_lsp", +) xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( xla, "com_github_grpc_grpc", + "eigen_archive", + "farmhash_archive", + "farmhash_gpu_archive", + "highwayhash", + "hwloc", "llvm-raw", "local_config_cuda", "local_config_remote_execution", "local_config_rocm", "local_config_sycl", "local_config_tensorrt", + "ml_dtypes_py", "python_version_repo", "rules_ml_toolchain", + "shardy", + "snappy", "stablehlo", "sycl_configure", "sycl_configure_ext", @@ -127,39 +354,22 @@ use_repo( ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") -llvm.configure() +llvm.configure(targets = [ + "X86", + "NVPTX", + "AMDGPU", + "AArch64", +]) use_repo(llvm, "llvm-project") -rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") -rust.toolchain( - edition = "2024", - extra_target_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], - versions = ["1.92.0"], -) -use_repo(rust, "rust_toolchains") - -register_toolchains("@rust_toolchains//:all") - -crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") -crate.from_cargo( - name = "crates", - cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", - cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", - platform_triples = [ - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - ], -) -use_repo(crate, "crates") - apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "apt_cpu", + lock = "//platforms/cpu:packages.lock.json", + manifest = "//platforms/cpu:packages.yaml", +) +use_repo(apt, "apt_cpu") + apt.install( name = "apt_cuda", lock = "//platforms/cuda:packages.lock.json", @@ -174,6 +384,14 @@ apt.install( ) use_repo(apt, "apt_rocm") +apt.install( + name = "apt_oneapi", + lock = "//platforms/oneapi:packages.lock.json", + manifest = "//platforms/oneapi:packages.yaml", + resolve_transitive = False, +) +use_repo(apt, "apt_oneapi") + apt.install( name = "apt_neuron", lock = "//platforms/neuron:packages.lock.json", @@ -188,10 +406,16 @@ oci.pull( image = "gcr.io/distroless/cc-debian12", platforms = ["linux/amd64"], ) -use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") - -git_override( - module_name = "rules_zig", - commit = "6c5d6a07b4ef48be894e7418bcae37bac54e1c2e", - remote = "https://github.com/zml/rules_zig.git", +oci.pull( + name = "debian_12", + digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + image = "mirror.gcr.io/library/debian:12", + platforms = ["linux/amd64"], +) +use_repo( + oci, + "debian_12", + "debian_12_linux_amd64", + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cc67df7a..1e746605 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,47 +1,55 @@ { - "lockFileVersion": 24, + "lockFileVersion": 28, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", - "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", - "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", - "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/MODULE.bazel": "e33b3801443f5fd64465262084534115db76363df13d2168a42bbfacc747be81", - "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/MODULE.bazel": "b4a76a8d70ba08aa0253fd5e374ca5a5fa6b15aba985a7bbe4467c138954830f", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/source.json": "a39a26fec1b7037bdd7ea6c25cc098b99ae80af10ff0b6ec8c4bfe0df694f451", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/MODULE.bazel": "5ebe5bf853769c65707e5c28f216798f7a4b1042015e6a36e6d03094d94bec8a", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/source.json": "0e8fc4f088ce07099c1cd6594c20c7ddbb48b4b3c0849b7d94ba94be88ff042b", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", + "https://bcr.bazel.build/modules/apple_support/1.13.0/MODULE.bazel": "7c8cdea7e031b7f9f67f0b497adf6d2c6a2675e9304ca93a9af6ed84eef5a524", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", + "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", + "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", + "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", + "https://bcr.bazel.build/modules/apple_support/1.22.1/MODULE.bazel": "90bd1a660590f3ceffbdf524e37483094b29352d85317060b2327fff8f3f4458", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", - "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", - "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", + "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", + "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", + "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", + "https://bcr.bazel.build/modules/apple_support/2.8.0/MODULE.bazel": "c45f5176057092afba0be6c48a9ea02101666802f8be536947e099e94757f8ed", + "https://bcr.bazel.build/modules/apple_support/2.8.0/source.json": "b7f6d87669c206adaa7d82cc44733349c63e341b80a0e049496ebd91d78c3472", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", + "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", + "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", + "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", - "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", + "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.13.0/MODULE.bazel": "c14c33c7c3c730612bdbe14ebbb5e61936b6f11322ea95a6e91cd1ba962f94df", "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", @@ -50,56 +58,155 @@ "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", - "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", + "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", + "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", + "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/MODULE.bazel": "2083ef9c7a469f520890483ccf8e0189d6e71e2117e7752e15e6554433d5ae3e", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/source.json": "e0ee3debde2789ff56e4452e612d126925ba9ab64d4bde79c67f099d2902df9b", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", - "https://bcr.bazel.build/modules/bazel_lib/3.2.0/source.json": "a6f45a903134bebbf33a6166dd42b4c7ab45169de094b37a85f348ca41170a84", + "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.0/MODULE.bazel": "2ab127ef8d56a739a99bb2ce00ec4c7d1ecc7977d4370c0ca6efd0d8f03d6d99", "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.0/MODULE.bazel": "2fb3fb53675f6adfc1ca5bfbd5cfb655ae350fba4706d924a8ec7e3ba945671c", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/MODULE.bazel": "396c1ef53835aafe3d42ce6619080531ee770648303731f16cfaa33fa056bf0c", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/source.json": "abaf8ac9d2ab2f47bda9af4c0c080ff7907378888e1f4bc62a0539dd13ba61e8", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/MODULE.bazel": "e76479eae70bd4e8f5f4c2dfc5d03ab971cfb18750246c7b3f3454c5c2ee6629", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/source.json": "9395c4679444bc47bf7e51a710366a4480aa371c6f6bed01868e2fabcf11acec", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20230215-5c22014/MODULE.bazel": "4b03dc0d04375fa0271174badcd202ed249870c8e895b26664fd7298abea7282", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20240530-2db0eb3/MODULE.bazel": "d0405b762c5e87cd445b7015f2b8da5400ef9a8dbca0bfefa6c1cea79d528a97", + "https://bcr.bazel.build/modules/boringssl/0.20240913.0/MODULE.bazel": "fcaa7503a5213290831a91ed1eb538551cf11ac0bc3a6ad92d0fef92c5bd25fb", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/MODULE.bazel": "b540cff73d948cb79cb0bc108d7cef391d2098a25adabfda5043e4ef548dbc87", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/source.json": "d843092e682b84188c043ac742965d7f96e04c846c7e338187e03238674909a9", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", + "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", + "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", + "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", + "https://bcr.bazel.build/modules/cel-spec/0.15.0/MODULE.bazel": "e1eed53d233acbdcf024b4b0bc1528116d92c29713251b5154078ab1348cb600", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/MODULE.bazel": "e310c7aff8490ed689ccafd32729b77a660b9547f5a5ba9b20e967011c324b36", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/source.json": "522d08bc22524e07863276dd0f038f446a83166e91281dcfc07d5b8433c8d89e", + "https://bcr.bazel.build/modules/civetweb/1.16/MODULE.bazel": "46a38f9daeb57392e3827fce7d40926be0c802bd23cdd6bfd3a96c804de42fae", + "https://bcr.bazel.build/modules/civetweb/1.16/source.json": "ba8b9585adb8355cb51b999d57172fd05e7a762c56b8d4bac6db42c99de3beb7", + "https://bcr.bazel.build/modules/curl/8.7.1/MODULE.bazel": "088221c35a2939c555e6e47cb31a81c15f8b59f4daa8009b1e9271a502d33485", + "https://bcr.bazel.build/modules/curl/8.8.0/MODULE.bazel": "7da3b3e79b0b4ee8f8c95d640bc6ad7b430ce66ef6e9c9d2bc29b3b5ef85f6fe", + "https://bcr.bazel.build/modules/curl/8.8.0/source.json": "d7d138b6878cf38891692fee0649ace35357fd549b425614d571786f054374d4", + "https://bcr.bazel.build/modules/cython/3.0.11-1/MODULE.bazel": "868b3f5c956c3657420d2302004c6bb92606bfa47e314bab7f2ba0630c7c966c", + "https://bcr.bazel.build/modules/cython/3.0.11-1/source.json": "da318be900b8ca9c3d1018839d3bebc5a8e1645620d0848fa2c696d4ecf7c296", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/source.json": "028519164a2e24563f4b43d810fdedc702daed90e71e7042d45ba82ad807b46f", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", + "https://bcr.bazel.build/modules/gazelle/0.39.1/MODULE.bazel": "1fa3fefad240e535066fd0e6950dfccd627d36dc699ee0034645e51dbde3980f", + "https://bcr.bazel.build/modules/gazelle/0.45.0/MODULE.bazel": "ecd19ebe9f8e024e1ccffb6d997cc893a974bcc581f1ae08f386bdd448b10687", + "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc", + "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", + "https://bcr.bazel.build/modules/gazelle/0.51.3/source.json": "fbe5312a01fb4a2a58caff4a0d40dcf384b6f0bda75e85546663360da1d7538a", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/source.json": "84590f7bc5a1fd99e1ef274ee16bb41c214f705e62847b42e705010dfa81fe53", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/MODULE.bazel": "cf01757e7590c56140a4b81638ff2b3e7074769e6271720bbf738fcda25b6fc2", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/source.json": "ab0e3a2ee9968a8848f59872fbbfa3e1f768597d71d2229e6caa319d357967c7", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/MODULE.bazel": "0a207a4c49da28c5cc1f7b3aeb23c2f7828c85c14aa8d9db0e30357a8d2250ed", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/source.json": "ef189be4e7853e1ebc6123fe20b71822bf9896bd1f8eed8f68505c4585f72a48", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/MODULE.bazel": "3553358a9d8d96026c9e28d9fb6c268574950d0be7fa9b4c0aeaf3c37c73f2d3", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/source.json": "fa7b79043b3c82bf74f1f2fa45af289e19b247375868d0752db2c114a1c7366c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/MODULE.bazel": "d633989337d069b5a95e6101777319681d7a4af4677e36801f11839d6512095c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/source.json": "ee59e2de37e4b531172870ac0296afa38f1ea004105ee21b2793c31a9d0ddccd", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/MODULE.bazel": "0ccd1614a914fb524b3ac267f9c97f9a5cd5412b027f0176b81a725882ec42ff", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/source.json": "24364f075ec5e6d5e0cfc5d651bd833b119e0cea1a45d51330c93461e3586e42", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/MODULE.bazel": "97c6a4d413b373d4cc97065da3de1b2166e22cbbb5f4cc9f05760bfa83619e24", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/source.json": "cf611c836a60e98e2e2ab2de8004f119e9f06878dcf4ea2d95a437b1b7a89fe9", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240326-1c8d509c5/MODULE.bazel": "a4b7e46393c1cdcc5a00e6f85524467c48c565256b22b5fae20f84ab4a999a68", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/MODULE.bazel": "117b7c7be7327ed5d6c482274533f2dbd78631313f607094d4625c28203cacdf", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20241220-5e258e33.bcr.1/MODULE.bazel": "ee6c30f82ecd476e61f019fb1151aaab380ea419958ff274ef2f0efca7969f5c", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/MODULE.bazel": "cc9e5ed294ed9ebf42cdbbdddd2df29048519e3797004df1e3f369f31ff4f2d4", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/source.json": "21558a194c519e27262cca9cf031bb166a666a8b7fb89993b700c828f8dc0857", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/MODULE.bazel": "827f54f492a3ce549c940106d73de332c2b30cebd0c20c0bc5d786aba7f116cb", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/source.json": "3664514073a819992320ffbce5825e4238459df344d8b01748af2208f8d2e1eb", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", - "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/grpc-java/1.62.2/MODULE.bazel": "99b8771e8c7cacb130170fed2a10c9e8fed26334a93e73b42d2953250885a158", + "https://bcr.bazel.build/modules/grpc-java/1.66.0/MODULE.bazel": "86ff26209fac846adb89db11f3714b3dc0090fb2fb81575673cc74880cda4e7e", + "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", + "https://bcr.bazel.build/modules/grpc/1.80.0/MODULE.bazel": "35ef253a0a31055c8a2495640516224647545b51673b0ccb956b116ee92ef554", + "https://bcr.bazel.build/modules/grpc/1.80.0/source.json": "55bf4cdcf278a450185ff93bef843a0d7db0a83d0b9761d9c1dee91faa9cdf46", + "https://bcr.bazel.build/modules/highway/1.4.0/MODULE.bazel": "22d48d947bda0c685d6174e62f9b6690089fefb87dfff4943cd420410ae82f68", + "https://bcr.bazel.build/modules/highway/1.4.0/source.json": "ae87bf2499d4902a15d760e14b7a1e94dc5dd0564bec256b9b0ed46c0e835f89", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/MODULE.bazel": "64f508885f907ac2518039b3d0c5c1703a8f6137d9f5d635067ba93d33c2670a", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/source.json": "13199fa0a267ca46814e9e6ab313a71890c8f1822e57376fdc23a2d2cd384051", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/MODULE.bazel": "4cd5b9355f1c2533c30f01904b71ac1b1a9bf61c43c81ba9972a5bf8f216476b", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", + "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", + "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", + "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", + "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/MODULE.bazel": "a1c8bb07b5b91d971727c635f449d05623ac9608f6fe4f5f04254ea12f08e349", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/source.json": "93f82a5ae985eb935c539bfee95e04767187818189241ac956f3ccadbdb8fb02", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/MODULE.bazel": "789706a714855f92c5c8cfcf1ef32bbb64dcd3b7c9066756ad7986ec59709d29", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/source.json": "aadf3f53e08b72376506b7c4ea3d167010c9efb160d7d6e1e304ed646bac1b36", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1/MODULE.bazel": "4a2e8b4d0b544002502474d611a5a183aa282251e14f6a01afe841c0c1b10372", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/MODULE.bazel": "49c0c07e8fb87b480bccb842cfee1b32617f11dac590f732573c69058699a3d1", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/source.json": "0c0872e048bbea052a9c541fb47019481a19201ba5555a71d762ad591bf94e1f", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/MODULE.bazel": "3455326c08b28415648a3d60d8e3c811847ebdbe64474f75b25878f25585aea1", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/source.json": "4e48137e4c3ecb99401ff99876df8fa330598d7da051869bec643446e8a8ff95", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.4.0.bcr.1/MODULE.bazel": "5ceaf25e11170d22eded4c8032728b4a3f273765fccda32f9e94f463755c4167", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/MODULE.bazel": "7543d91a53b98e7b5b37c5a0865b93bff12c1ee022b1e322cd236b968894b030", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/source.json": "046b721ce203e88cdaad44d7dd17a86b7200eab9388b663b234e72e13ff7b143", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", + "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/source.json": "50639625e937b56115012674c797cca7a05a96b4878c87d803c13dc2b31de8a0", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -113,28 +220,63 @@ "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", - "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", + "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", + "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0/MODULE.bazel": "ce82e086bbc0b60267e970f6a54b2ca6d0f22d3eb6633e00e2cc2899c700f3d8", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", + "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", + "https://bcr.bazel.build/modules/protobuf/25.6/MODULE.bazel": "fc0ae073b47c7ede88b825ff79e64f1c058967c7a87a86cdf4abecd9e0516625", + "https://bcr.bazel.build/modules/protobuf/26.0/MODULE.bazel": "8402da964092af40097f4a205eec2a33fd4a7748dc43632b7d1629bfd9a2b856", + "https://bcr.bazel.build/modules/protobuf/27.0-rc2/MODULE.bazel": "b2b0dbafd57b6bec0ca9b251da02e628c357dab53a097570aa7d79d020f107cf", "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", + "https://bcr.bazel.build/modules/protobuf/27.2/MODULE.bazel": "32450b50673882e4c8c3d10a83f3bc82161b213ed2f80d17e38bece8f165c295", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/29.3/MODULE.bazel": "77480eea5fb5541903e49683f24dc3e09f4a79e0eea247414887bb9fc0066e94", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/33.1/MODULE.bazel": "982c8a0cceab4d790076f72b7677faf836b0dfadc2b66a34aab7232116c4ae39", - "https://bcr.bazel.build/modules/protobuf/33.1/source.json": "992c237a40899425648213bf79b05f08c6e8dcd619f96cd944b4511b0276fbd8", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", + "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", + "https://bcr.bazel.build/modules/protobuf/30.0/MODULE.bazel": "0e736de5d52ad7824113f47e65256a26ee74b689ba859c5447a0663e5a075409", + "https://bcr.bazel.build/modules/protobuf/31.1/MODULE.bazel": "379a389bb330b7b8c1cdf331cc90bf3e13de5614799b3b52cdb7c6f389f6b38e", + "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", + "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", + "https://bcr.bazel.build/modules/protobuf/35.0/MODULE.bazel": "ee97170c013d94c366bbb5fe8a97b0650477cbc00e5e0f6a2c297484bc34d81b", + "https://bcr.bazel.build/modules/protobuf/35.0/source.json": "2162e80adef862606ba85ae42e0df85eb519376fd6ab49caddee6fb776a1281e", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4/MODULE.bazel": "b8913c154b16177990f6126d2d2477d187f9ddc568e95ee3e2d50fc65d2c494a", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/MODULE.bazel": "4bf09676b62fa587ae07e073420a76ec8766dcce7545e5f8c68cfa8e484b5120", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/source.json": "c19071ebc4b53b5f1cfab9c66eefaf6e4179eb8a998970d07b1077687e777f29", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", - "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", + "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/MODULE.bazel": "67899804c3dbdf169d107ce7c00e60a38a46d097c7ff3460bdebe192cbc77f56", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/source.json": "5bcb3ae55c0d5a159bd910154d7ee8873bdf61dd7b13cd0a2bb456011ec706ae", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/MODULE.bazel": "82fbcb2e42f9e0040e76ccc74c06c3e46dfd33c64ca359293f8b84df0e6dff4c", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/source.json": "5c42389ad0e21fc06b95ad7c0b730008271624a2fa3292e0eab5f30e15adeee3", + "https://bcr.bazel.build/modules/re2/2021-09-01/MODULE.bazel": "bcb6b96f3b071e6fe2d8bed9cc8ada137a105f9d2c5912e91d27528b3d123833", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", + "https://bcr.bazel.build/modules/re2/2024-05-01/MODULE.bazel": "55a3f059538f381107824e7d00df5df6d061ba1fb80e874e4909c0f0549e8f3e", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", - "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/re2/2025-08-12.bcr.1/MODULE.bazel": "e09b434b122bfb786a69179f9b325e35cb1856c3f56a7a81dd61609260ed46e1", + "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", + "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", - "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_android/0.7.1/MODULE.bazel": "a806fc382a774252f228a40e3b11b9fcc6276f8778c7fb33e9f72937c6258363", + "https://bcr.bazel.build/modules/rules_android/0.7.1/source.json": "151440aed3f0f73a00d4ed5cec5d31f63a6fef9b95d8fab1eb1810150fa525f2", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", + "https://bcr.bazel.build/modules/rules_buf/0.1.1/MODULE.bazel": "6189aec18a4f7caff599ad41b851ab7645d4f1e114aa6431acf9b0666eb92162", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -143,92 +285,156 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.5/MODULE.bazel": "be41f87587998fe8890cd82ea4e848ed8eb799e053c224f78f3ff7fe1a1d9b74", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", + "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", - "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", + "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", + "https://bcr.bazel.build/modules/rules_cc/0.2.20/MODULE.bazel": "f5c07bce5ddcb99be21a0812ff5aadb439e688b7449c6542152363b2fd859c1a", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/MODULE.bazel": "4a3d4f3606d3b3190be495dbc497acca552807d1d5540661ef0d1658472882e3", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", - "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", - "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.45.1/MODULE.bazel": "6d7884f0edf890024eba8ab31a621faa98714df0ec9d512389519f0edff0281a", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", + "https://bcr.bazel.build/modules/rules_go/0.49.0/MODULE.bazel": "61cfc1ba17123356d1b12b6c50f6e0162b2cc7fd6f51753c12471e973a0f72a5", + "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", + "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a", + "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", + "https://bcr.bazel.build/modules/rules_go/0.62.0/MODULE.bazel": "8ee616065c3d2b2f7ac0880108316ce8d0c332b3a30aad24e95c0bc124ec853e", + "https://bcr.bazel.build/modules/rules_go/0.62.0/source.json": "36d781c558eb7d3bd49dc5e190455714f174232372f15207ab200b4348bda3e6", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", + "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615", - "https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc", + "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", + "https://bcr.bazel.build/modules/rules_java/8.9.0/MODULE.bazel": "e17c876cb53dcd817b7b7f0d2985b710610169729e8c371b2221cacdcd3dce4a", + "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", + "https://bcr.bazel.build/modules/rules_java/9.3.0/MODULE.bazel": "f657c72d65ac449caae9abf2e68e66c0d36f9416848c4c4903d0b3234229e7f2", + "https://bcr.bazel.build/modules/rules_java/9.3.0/source.json": "59ae7e662c3c7042b88bbb42ad12483523e234c65ebe4c51611baa43e85cb248", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", + "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.2/MODULE.bazel": "36a6e52487a855f33cb960724eb56547fa87e2c98a0474c3acad94339d7f8e99", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", - "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/MODULE.bazel": "07c5db05527db7744a54fcffd653e1550d40e0540207a7f7e6d0a4de5bef8274", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/source.json": "b12970214f3cc144b26610caeb101fa622d910f1ab3d98f0bae1058edbd00bd4", "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.5/MODULE.bazel": "043a16a572f610558ec2030db3ff0c9938574e7dd9f58bded1bb07c0192ef025", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/MODULE.bazel": "3443d53d275e14fecfebd0b491f01d06ea3883c04a1b3336e7ae9d5ec9066bef", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/source.json": "5a5553cffea43f2c5156c8ad0de4a14ad95413ceb39cd4d08f50e2aea86927e8", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.20.0/MODULE.bazel": "bfe14d17f20e3fe900b9588f526f52c967a6f281e47a1d6b988679bd15082286", + "https://bcr.bazel.build/modules/rules_python/0.22.0/MODULE.bazel": "b8057bafa11a9e0f4b08fc3b7cd7bee0dcbccea209ac6fc9a3ff051cd03e19e9", + "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", + "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", + "https://bcr.bazel.build/modules/rules_python/0.35.0/MODULE.bazel": "c3657951764cdcdb5a7370d5e885fad5e8c1583320aad18d46f9f110d2c22755", + "https://bcr.bazel.build/modules/rules_python/0.37.1/MODULE.bazel": "3faeb2d9fa0a81f8980643ee33f212308f4d93eea4b9ce6f36d0b742e71e9500", + "https://bcr.bazel.build/modules/rules_python/0.37.2/MODULE.bazel": "b5ffde91410745750b6c13be1c5dc4555ef5bc50562af4a89fd77807fdde626a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", + "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", - "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", - "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", - "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", - "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", - "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", + "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", + "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", + "https://bcr.bazel.build/modules/rules_python/1.8.0/MODULE.bazel": "c151c025dbcc93d8f62ab68ecc313c9176a868a0e6386981bf2a12aec77cbe7b", + "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", + "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/MODULE.bazel": "d44fec647d0aeb67b9f3b980cf68ba634976f3ae7ccd6c07d790b59b87a4f251", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/source.json": "37c10335f2361c337c5c1f34ed36d2da70534c23088062b33a8bdaab68aa9dea", + "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/MODULE.bazel": "4326fec48f2fef0d514de46346f7f77e200c82936dd08b91c9ef039fbdad5c10", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/source.json": "0d094307d690cc18b3ab003998697be8070a206f65592c5c8476999796f11c4b", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.5.0/MODULE.bazel": "6d2fec72be0ad2e1fc2eab536cf9d3fb149037c6d2aa678078f3512c06a819e1", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", + "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -239,35 +445,53 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", - "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", - "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", + "https://bcr.bazel.build/modules/windows_support/0.4.1/MODULE.bazel": "2287f95ac65c48ea05197f626a9b5711ea3c88cea61f2a5782882bdd55fdeb51", + "https://bcr.bazel.build/modules/windows_support/0.4.1/source.json": "bdb72e6717b00135e0a4a8512b3f6b626df0baea0bde74bf7312c7484f791287", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/source.json": "4666d3035f69063ecd136f3f0c68e424f1b1dd97ffc9cceb679522c0be38761b", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", + "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/MODULE.bazel": "03e1c50b16fc2403de5dd98ff8e2dc0b404865740434b8705c7c0853241f943d", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/source.json": "69b87f3bff38ba29399c012f3e3fc0f239ac4e5c1ae66ebd9481be6d8461da55", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/MODULE.bazel": "ff7cf8a5334a96b7b9ffcbe73418d45929ce125efebac39c1ee7d14bd2ae7a9b", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/source.json": "abf61f3c1630eab7b81722f710060b6f31e7c160511bcbfaad74bb4385883051", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/MODULE.bazel": "86130924115bf8dfc4aba286fb49a2ec8d32430a21771db9ccf5381acf0d10cc", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/source.json": "7aa4112eb2fbb99757e97cf0b7a4e2bdb3464b2b3cfb351bc807ffcb34bd349f", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/MODULE.bazel": "c5977176dd8555be7a9d598512ae0cae11831259c06f00a5e5e0037d5db4e3f5", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/source.json": "aa95e0b5aac9d80195b9047d223beaf26b1948be55d49f0e803e180e9ccc6e75" }, "selectedYankedVersions": {}, "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", + "bzlTransitiveDigest": "6o8lyMtImPc3Si+zcGss6WH8IY80mTWhC0MC3r0ra8Y=", "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "recordedInputs": [ + "REPO_MAPPING:aspect_rules_py+,aspect_tools_telemetry_report aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report", + "REPO_MAPPING:aspect_rules_py+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" + ], "generatedRepoSpecs": { "rules_py_tools.darwin_amd64": { "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", @@ -309,130 +533,54 @@ "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" } } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_rules_py+", - "aspect_tools_telemetry_report", - "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" - ], - [ - "aspect_rules_py+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "aspect_rules_py+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", + "recordedInputs": [ + "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", + "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" + ], "generatedRepoSpecs": { "aspect_tools_telemetry_report": { "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { "aspect_rules_py": "1.8.4", - "rules_rs": "0.0.28", "aspect_tools_telemetry": "0.3.3" } } } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_tools_telemetry+", - "bazel_lib", - "bazel_lib+" - ], - [ - "aspect_tools_telemetry+", - "bazel_skylib", - "bazel_skylib+" - ] - ] + } } }, - "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "@@rules_android+//rules/android_sdk_repository:rule.bzl%android_sdk_repository_extension": { "general": { - "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", - "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "aLDGLKolYFwLsBssOQfwOs+bo3Aof0yKkW0HQRPX3ms=", + "usagesDigest": "iEGI2aNDMkHt9LXCdViLNUUOslpiVj2DrevWWXZEFnU=", + "recordedInputs": [], "generatedRepoSpecs": { - "com_github_jetbrains_kotlin_git": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", - "attributes": { - "urls": [ - "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" - ], - "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" - } - }, - "com_github_jetbrains_kotlin": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", - "attributes": { - "git_repository_name": "com_github_jetbrains_kotlin_git", - "compiler_version": "1.9.23" - } - }, - "com_github_google_ksp": { - "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", - "attributes": { - "urls": [ - "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" - ], - "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", - "strip_version": "1.9.23-1.0.20" - } - }, - "com_github_pinterest_ktlint": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", - "attributes": { - "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", - "urls": [ - "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" - ], - "executable": true - } - }, - "rules_android": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", - "strip_prefix": "rules_android-0.1.1", - "urls": [ - "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" - ] - } + "androidsdk": { + "repoRuleId": "@@rules_android+//rules/android_sdk_repository:rule.bzl%_android_sdk_repository", + "attributes": {} } - }, - "recordedRepoMappingEntries": [ - [ - "rules_kotlin+", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "nCIRDeZJX85AXhvovlzr0Sx1f3ZKx8IS8xvTHWlUlqM=", - "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "cbfE3+b9HnO8wGMMbdSzaHKjcON7SL8gb4fKTNiVHug=", + "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", + "recordedInputs": [ + "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", + "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", + "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", + "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" + ], "generatedRepoSpecs": { "distroless_cc_debian12_linux_amd64": { "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", @@ -463,6 +611,35 @@ "reproducible": true } }, + "debian_12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "mirror.gcr.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platform": "linux/amd64", + "target_name": "debian_12_linux_amd64", + "bazel_tags": [] + } + }, + "debian_12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "debian_12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "mirror.gcr.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platforms": { + "@@platforms//cpu:x86_64": "@debian_12_linux_amd64" + }, + "bzlmod_repository": "debian_12", + "reproducible": true + } + }, "oci_crane_darwin_amd64": { "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { @@ -580,370 +757,111 @@ "moduleExtensionMetadata": { "explicitRootModuleDirectDeps": [ "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64" + "distroless_cc_debian12_linux_amd64", + "debian_12", + "debian_12_linux_amd64" ], "explicitRootModuleDirectDevDeps": [], "useAllRepos": "NO", "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib+", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci+", - "aspect_bazel_lib", - "aspect_bazel_lib+" - ], - [ - "rules_oci+", - "bazel_features", - "bazel_features+" - ], - [ - "rules_oci+", - "bazel_skylib", - "bazel_skylib+" - ] - ] - } - }, - "@@rules_python+//python/extensions:config.bzl%config": { - "general": { - "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", - "usagesDigest": "p2al+dDKI5UlCyNvheMVynbWSGbdiji/jMz53fMNfJA=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "rules_python_internal": { - "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", - "attributes": { - "transition_setting_generators": {}, - "transition_settings": [] - } - }, - "pypi__build": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", - "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__click": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__colorama": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__importlib_metadata": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", - "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__installer": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__more_itertools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", - "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__packaging": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", - "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pep517": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", - "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip_tools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", - "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pyproject_hooks": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", - "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__setuptools": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", - "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__tomli": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__wheel": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", - "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__zipp": { - "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", - "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "pypi__build", - "rules_python++config+pypi__build" - ], - [ - "rules_python+", - "pypi__click", - "rules_python++config+pypi__click" - ], - [ - "rules_python+", - "pypi__colorama", - "rules_python++config+pypi__colorama" - ], - [ - "rules_python+", - "pypi__importlib_metadata", - "rules_python++config+pypi__importlib_metadata" - ], - [ - "rules_python+", - "pypi__installer", - "rules_python++config+pypi__installer" - ], - [ - "rules_python+", - "pypi__more_itertools", - "rules_python++config+pypi__more_itertools" - ], - [ - "rules_python+", - "pypi__packaging", - "rules_python++config+pypi__packaging" - ], - [ - "rules_python+", - "pypi__pep517", - "rules_python++config+pypi__pep517" - ], - [ - "rules_python+", - "pypi__pip", - "rules_python++config+pypi__pip" - ], - [ - "rules_python+", - "pypi__pip_tools", - "rules_python++config+pypi__pip_tools" - ], - [ - "rules_python+", - "pypi__pyproject_hooks", - "rules_python++config+pypi__pyproject_hooks" - ], - [ - "rules_python+", - "pypi__setuptools", - "rules_python++config+pypi__setuptools" - ], - [ - "rules_python+", - "pypi__tomli", - "rules_python++config+pypi__tomli" - ], - [ - "rules_python+", - "pypi__wheel", - "rules_python++config+pypi__wheel" - ], - [ - "rules_python+", - "pypi__zipp", - "rules_python++config+pypi__zipp" - ] - ] + } } }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", - "usagesDigest": "lPQI4kxB6q1JLGHa7OFxCNrWxnLjeMKg5ogOIB5OF6U=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "ZtcPw94gqLP8+ocBKipnnyLIivxEC/OUYen7Kx7hnEA=", + "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,platforms platforms" + ], "generatedRepoSpecs": { - "uv_0_6_5_aarch64_apple_darwin": { + "uv_0_9_30_aarch64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "aarch64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" ], - "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" + "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" } }, - "uv_0_6_5_aarch64_unknown_linux_gnu": { + "uv_0_9_30_aarch64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "aarch64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" ], - "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" + "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" } }, - "uv_0_6_5_powerpc64_unknown_linux_gnu": { + "uv_0_9_30_powerpc64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "powerpc64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" ], - "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" + "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" } }, - "uv_0_6_5_powerpc64le_unknown_linux_gnu": { + "uv_0_9_30_powerpc64le_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "powerpc64le-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" ], - "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" + "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" } }, - "uv_0_6_5_s390x_unknown_linux_gnu": { + "uv_0_9_30_s390x_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "s390x-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" ], - "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" + "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" } }, - "uv_0_6_5_x86_64_apple_darwin": { + "uv_0_9_30_x86_64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" ], - "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" + "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" } }, - "uv_0_6_5_x86_64_pc_windows_msvc": { + "uv_0_9_30_x86_64_pc_windows_msvc": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-pc-windows-msvc", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" ], - "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" + "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" } }, - "uv_0_6_5_x86_64_unknown_linux_gnu": { + "uv_0_9_30_x86_64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.6.5", + "version": "0.9.30", "platform": "x86_64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" ], - "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" + "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" } }, "uv": { @@ -951,55 +869,55 @@ "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "0_6_5_aarch64_apple_darwin", - "0_6_5_aarch64_unknown_linux_gnu", - "0_6_5_powerpc64_unknown_linux_gnu", - "0_6_5_powerpc64le_unknown_linux_gnu", - "0_6_5_s390x_unknown_linux_gnu", - "0_6_5_x86_64_apple_darwin", - "0_6_5_x86_64_pc_windows_msvc", - "0_6_5_x86_64_unknown_linux_gnu" + "0_9_30_aarch64_apple_darwin", + "0_9_30_aarch64_unknown_linux_gnu", + "0_9_30_powerpc64_unknown_linux_gnu", + "0_9_30_powerpc64le_unknown_linux_gnu", + "0_9_30_s390x_unknown_linux_gnu", + "0_9_30_x86_64_apple_darwin", + "0_9_30_x86_64_pc_windows_msvc", + "0_9_30_x86_64_unknown_linux_gnu" ], "toolchain_implementations": { - "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", - "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", - "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", - "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", - "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" + "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", + "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", + "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", + "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", + "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" }, "toolchain_compatible_with": { - "0_6_5_aarch64_apple_darwin": [ + "0_9_30_aarch64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:aarch64'" ], - "0_6_5_aarch64_unknown_linux_gnu": [ + "0_9_30_aarch64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:aarch64'" ], - "0_6_5_powerpc64_unknown_linux_gnu": [ + "0_9_30_powerpc64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc'" ], - "0_6_5_powerpc64le_unknown_linux_gnu": [ + "0_9_30_powerpc64le_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc64le'" ], - "0_6_5_s390x_unknown_linux_gnu": [ + "0_9_30_s390x_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:s390x'" ], - "0_6_5_x86_64_apple_darwin": [ + "0_9_30_x86_64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:x86_64'" ], - "0_6_5_x86_64_pc_windows_msvc": [ + "0_9_30_x86_64_pc_windows_msvc": [ "'@@platforms//os:windows'", "'@@platforms//cpu:x86_64'" ], - "0_6_5_x86_64_unknown_linux_gnu": [ + "0_9_30_x86_64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:x86_64'" ] @@ -1007,36 +925,24 @@ "toolchain_target_settings": {} } } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python+", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_python+", - "platforms", - "platforms" - ] - ] + } } }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", - "usagesDigest": "8TQ1JYfn2xQJIQya4EwCvLiuZAdGIocYzOMc1au4GNs=", - "recordedFileInputs": { - "@@//bazel/zig_index.json": "0b2647eea9809f05c33dac461c6572e85e0401221bca32ebb6353100132f29c5", - "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "hPjDj28qKZsSFAE45ZtQPcgDkeRXAcwxiuAQyeEF7ng=", + "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", + "recordedInputs": [ + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", + "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" + ], "generatedRepoSpecs": { - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux": { + "zig_0.17.0_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1051,17 +957,30 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "zig_version": "0.17.0", "platform": "aarch64-linux" } }, - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos": { + "zig_0.17.0_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1076,17 +995,30 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "zig_version": "0.17.0", "platform": "aarch64-macos" } }, - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows": { + "zig_0.17.0_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1101,17 +1033,30 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "zig_version": "0.17.0", "platform": "aarch64-windows" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux": { + "zig_0.17.0_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1126,17 +1071,30 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "zig_version": "0.17.0", "platform": "x86_64-linux" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos": { + "zig_0.17.0_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1151,17 +1109,30 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "zig_version": "0.17.0", "platform": "x86_64-macos" } }, - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows": { + "zig_0.17.0_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1176,10 +1147,23 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/" + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" ], - "sha256": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", - "zig_version": "0.16.0-dev.2611+f996d2866", + "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "zig_version": "0.17.0", "platform": "x86_64-windows" } }, @@ -1187,75 +1171,108 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux", - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos", - "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos", - "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows" + "zig_0.17.0_aarch64-linux", + "zig_0.17.0_aarch64-macos", + "zig_0.17.0_aarch64-windows", + "zig_0.17.0_x86_64-linux", + "zig_0.17.0_x86_64-macos", + "zig_0.17.0_x86_64-windows" ], "labels": [ - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos//:zig_toolchain", - "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows//:zig_toolchain" + "@zig_0.17.0_aarch64-linux//:zig_toolchain", + "@zig_0.17.0_aarch64-macos//:zig_toolchain", + "@zig_0.17.0_aarch64-windows//:zig_toolchain", + "@zig_0.17.0_x86_64-linux//:zig_toolchain", + "@zig_0.17.0_x86_64-macos//:zig_toolchain", + "@zig_0.17.0_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866", - "0.16.0-dev.2611+f996d2866" + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0" ], "exec_lengths": [ - 2, - 2, - 2, - 2, - 2, - 2 + 3, + 3, + 3, + 3, + 3, + 3 ], "exec_constraints": [ "@platforms//os:linux", "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", "@platforms//os:macos", "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", "@platforms//os:windows", "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", "@platforms//os:linux", "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", "@platforms//os:macos", "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", "@platforms//os:windows", - "@platforms//cpu:x86_64" - ] + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'" + ], + "target_compatible_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_compatible_constraints": [], + "target_settings_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_settings": [] } } - }, - "recordedRepoMappingEntries": [ - [ - "rules_zig+", - "bazel_skylib", - "bazel_skylib+" - ], - [ - "rules_zig+", - "bazel_tools", - "bazel_tools" - ] - ] + } + } + }, + "@@rules_zig+//zig/zls:extensions.bzl%zls": { + "general": { + "bzlTransitiveDigest": "8UokkNRK4sShFNMJ3aDabiCS4Q/ldgChV/NYNszfmCg=", + "usagesDigest": "kOrP52crJGijpHNzo4S7msXvbGpfmzvkE4tuhpRWTsI=", + "recordedInputs": [ + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@rules_zig+//zig/zls/private/versions.json 127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" + ], + "generatedRepoSpecs": { + "zls_toolchains": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [], + "labels": [], + "zig_versions": [], + "exec_lengths": [], + "exec_constraints": [] + } + } + } } }, "@@yq.bzl+//yq:extensions.bzl%yq": { "general": { "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", - "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "usagesDigest": "1CP5kLHwnlFZ3obmnV0eEOp+80JltkcSimeCHXe8/+E=", + "recordedInputs": [], "generatedRepoSpecs": { "yq_darwin_amd64": { "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", @@ -1319,91 +1336,815 @@ "user_repository_name": "yq" } } - }, - "recordedRepoMappingEntries": [] + } } } }, "facts": { - "@@rules_rs+//rs:extensions.bzl%crate": { - "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", - "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", - "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", - "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", - "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", - "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", - "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", - "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", - "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", - "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", - "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", - "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", - "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", - "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", - "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", - "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", - "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", - "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", - "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", - "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", - "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", - "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", - "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", - "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", - "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", - "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", - "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", - "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", - "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", - "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", - "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", - "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", - "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", - "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", - "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", - "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", - "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", - "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", - "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", - "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", - "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", - "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", - "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", - "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", - "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", - "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", - "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", - "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", - "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", - "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", - "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", - "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", - "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", - "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", - "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", - "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", - "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", - "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", - "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", - "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", - "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", - "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", - "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", - "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", - "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", - "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", - "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", - "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", - "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", - "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", - "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", - "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", - "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" + "@@rules_go+//go:extensions.bzl%go_sdk": { + "1.21.1": { + "aix_ppc64": [ + "go1.21.1.aix-ppc64.tar.gz", + "ad5c160e2ff7989f818572f2391ca000d587569ff68d8591a695382a863719d0" + ], + "darwin_amd64": [ + "go1.21.1.darwin-amd64.tar.gz", + "809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19" + ], + "darwin_arm64": [ + "go1.21.1.darwin-arm64.tar.gz", + "ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1" + ], + "dragonfly_amd64": [ + "go1.21.1.dragonfly-amd64.tar.gz", + "646f5db8ba479032aea07c597ccd187bc75c161f18e7a1785c62ab8cb103cb3e" + ], + "freebsd_386": [ + "go1.21.1.freebsd-386.tar.gz", + "9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01" + ], + "freebsd_amd64": [ + "go1.21.1.freebsd-amd64.tar.gz", + "2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1" + ], + "freebsd_arm64": [ + "go1.21.1.freebsd-arm64.tar.gz", + "4946118e72deccf194a9bfcfedbafe52e45b5fe9441fd3acfced760b27a20f9a" + ], + "freebsd_armv6l": [ + "go1.21.1.freebsd-arm.tar.gz", + "6d4dab314ed6adff68a74e3bce516e36fd89c7bf4523c2c105f96e04933fc982" + ], + "freebsd_riscv64": [ + "go1.21.1.freebsd-riscv64.tar.gz", + "591bd28e2dac581facb20c1529f8c29590d7eaaccbcdc93d12a234461b7bfe89" + ], + "illumos_amd64": [ + "go1.21.1.illumos-amd64.tar.gz", + "829548b591675df5f113317ac011bfc3381da7a96bd6a1c748694370818b4497" + ], + "linux_386": [ + "go1.21.1.linux-386.tar.gz", + "b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901" + ], + "linux_amd64": [ + "go1.21.1.linux-amd64.tar.gz", + "b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae" + ], + "linux_arm64": [ + "go1.21.1.linux-arm64.tar.gz", + "7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967" + ], + "linux_armv6l": [ + "go1.21.1.linux-armv6l.tar.gz", + "f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85" + ], + "linux_loong64": [ + "go1.21.1.linux-loong64.tar.gz", + "2273cc4ed398d8b341aee01fd62f3845045b2f439b84b28720a7f3d3552f5d41" + ], + "linux_mips": [ + "go1.21.1.linux-mips.tar.gz", + "2ba529210bae7ba7b8c48b0084f713b41e8c109ae75637820fd5b72be9f3faff" + ], + "linux_mips64": [ + "go1.21.1.linux-mips64.tar.gz", + "f2cef18856b17cc5c81cc8b0a35b1ca08d2b04d4b88adc07d190560cb464e2f6" + ], + "linux_mips64le": [ + "go1.21.1.linux-mips64le.tar.gz", + "3aa007a41f533b50eae2491bbd29926ada09357367a8aa05e7e50ec50c78acf9" + ], + "linux_mipsle": [ + "go1.21.1.linux-mipsle.tar.gz", + "c0059f49c3ea31bebe972f47d460d49addb78641081a7000e79e730464bcce95" + ], + "linux_ppc64": [ + "go1.21.1.linux-ppc64.tar.gz", + "6871fd3e6dae3897112dc4a98988406461ef1018f4a17c87689d977eba8dbf20" + ], + "linux_ppc64le": [ + "go1.21.1.linux-ppc64le.tar.gz", + "eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7" + ], + "linux_riscv64": [ + "go1.21.1.linux-riscv64.tar.gz", + "fac64ed26e003f49f1d77f6d2c4cf951422aecbce12232d9ec1bf4585fc44ee1" + ], + "linux_s390x": [ + "go1.21.1.linux-s390x.tar.gz", + "a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d" + ], + "netbsd_386": [ + "go1.21.1.netbsd-386.tar.gz", + "c919f2ba4755e6aebeea6e7f19abf1c5ed13d37c2930867d10a86adfd0686eba" + ], + "netbsd_amd64": [ + "go1.21.1.netbsd-amd64.tar.gz", + "b2dd9913063caa2b5f960e8ebc6c225293b4155e9ebd88d43afd55bb81ca8931" + ], + "netbsd_arm64": [ + "go1.21.1.netbsd-arm64.tar.gz", + "5d245ac932d421389c1bfcedc38733367ae407dbdc162a6e7bfca5677fb15269" + ], + "netbsd_armv6l": [ + "go1.21.1.netbsd-arm.tar.gz", + "4d1756a9845917b07c8ec4999b8e019a2bd2a8b3d5d16da5da0ef3ce84d2139c" + ], + "openbsd_386": [ + "go1.21.1.openbsd-386.tar.gz", + "ce094e551227522aaf3111a94a7f115c5a2b4f862c42d349026503544a40fb96" + ], + "openbsd_amd64": [ + "go1.21.1.openbsd-amd64.tar.gz", + "321c31c42131db2bd0e9d67dd98450f55b281e8e6d8eb681594c39e6688ccbb2" + ], + "openbsd_arm64": [ + "go1.21.1.openbsd-arm64.tar.gz", + "4d65bcb383895be8d6c05507f2b410e15448b0610e1f5fb23ece9ff4e045f7a8" + ], + "openbsd_armv6l": [ + "go1.21.1.openbsd-arm.tar.gz", + "481254b99b9861b65c9eaab137af15311fca618f9f17f2bd872795368c726666" + ], + "plan9_386": [ + "go1.21.1.plan9-386.tar.gz", + "93ea25cbb8063cd4d5917b8ca58a72c26b6cb9e6c1d785e30867ae8ad8290484" + ], + "plan9_amd64": [ + "go1.21.1.plan9-amd64.tar.gz", + "14d61633059651d12cc02b2f53739016f1b07063a4140147e44a196413950553" + ], + "plan9_armv6l": [ + "go1.21.1.plan9-arm.tar.gz", + "adfacbcfb5e7415802187fb6aa8b5397d589fdedcd5d0eac3b86352da7390603" + ], + "solaris_amd64": [ + "go1.21.1.solaris-amd64.tar.gz", + "b86a223e52e41de3ff180f90f1d7e705d30f96271f9a81005e20f25be2658990" + ], + "windows_386": [ + "go1.21.1.windows-386.zip", + "170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b" + ], + "windows_amd64": [ + "go1.21.1.windows-amd64.zip", + "10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6" + ], + "windows_arm64": [ + "go1.21.1.windows-arm64.zip", + "41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef" + ], + "windows_armv6l": [ + "go1.21.1.windows-arm.zip", + "d7a2472999a80dbf6bbcb6783a2b4299916055473263117295022f18e7fad487" + ] + }, + "1.22.0": { + "aix_ppc64": [ + "go1.22.0.aix-ppc64.tar.gz", + "190e105fc4133a8b5bb1492f368fa89aa4b729270441120714be7ee82e871ebc" + ], + "darwin_amd64": [ + "go1.22.0.darwin-amd64.tar.gz", + "ebca81df938d2d1047cc992be6c6c759543cf309d401b86af38a6aed3d4090f4" + ], + "darwin_arm64": [ + "go1.22.0.darwin-arm64.tar.gz", + "bf8e388b09134164717cd52d3285a4ab3b68691b80515212da0e9f56f518fb1e" + ], + "dragonfly_amd64": [ + "go1.22.0.dragonfly-amd64.tar.gz", + "357ab446200effa26c73ccaf3e8551426428950bf2490859fb296a09e53228b1" + ], + "freebsd_386": [ + "go1.22.0.freebsd-386.tar.gz", + "b8065da37783e8b9e7086365a54d74537e832c92311b61101a66989ab2458d8e" + ], + "freebsd_amd64": [ + "go1.22.0.freebsd-amd64.tar.gz", + "50f421c7f217083ac94aab1e09400cb9c2fea7d337679ec11f1638a11460da30" + ], + "freebsd_arm64": [ + "go1.22.0.freebsd-arm64.tar.gz", + "e23385e5c640787fa02cd58f2301ea09e162c4d99f8ca9fa6d52766f428a933d" + ], + "freebsd_armv6l": [ + "go1.22.0.freebsd-arm.tar.gz", + "c9c8b305f90903536f4981bad9f029828c2483b3216ca1783777344fbe603f2d" + ], + "freebsd_riscv64": [ + "go1.22.0.freebsd-riscv64.tar.gz", + "c8f94d1de6024546194d58e7b9370dc7ea06176aad94a675b0062c25c40cb645" + ], + "illumos_amd64": [ + "go1.22.0.illumos-amd64.tar.gz", + "d6792f11ad6ee5fc42d2fe51e1f1683471aa2ee4f20e3ad1be22a4afdbd38e7f" + ], + "linux_386": [ + "go1.22.0.linux-386.tar.gz", + "1e209c4abde069067ac9afb341c8003db6a210f8173c77777f02d3a524313da3" + ], + "linux_amd64": [ + "go1.22.0.linux-amd64.tar.gz", + "f6c8a87aa03b92c4b0bf3d558e28ea03006eb29db78917daec5cfb6ec1046265" + ], + "linux_arm64": [ + "go1.22.0.linux-arm64.tar.gz", + "6a63fef0e050146f275bf02a0896badfe77c11b6f05499bb647e7bd613a45a10" + ], + "linux_armv6l": [ + "go1.22.0.linux-armv6l.tar.gz", + "0525f92f79df7ed5877147bce7b955f159f3962711b69faac66bc7121d36dcc4" + ], + "linux_loong64": [ + "go1.22.0.linux-loong64.tar.gz", + "b4b1d425cc113608452a32768469b6e34e538fd072bde9f508a75c8dbbdb843f" + ], + "linux_mips": [ + "go1.22.0.linux-mips.tar.gz", + "ec0f9194df750c78492c02e4c70ffe6f3864f2511e47c894415320db752719f8" + ], + "linux_mips64": [ + "go1.22.0.linux-mips64.tar.gz", + "47e938d215e4968ab42afb7307792e5e64184a717e8f176d0de7c411af96d63d" + ], + "linux_mips64le": [ + "go1.22.0.linux-mips64le.tar.gz", + "c162a1a516b8bc8839fc0c0283ad90f6b511e5aca3da4939ed4800f124b9e72c" + ], + "linux_mipsle": [ + "go1.22.0.linux-mipsle.tar.gz", + "6ce3e9a06e3a8ca0312dc1f85046b6914c19852eba5393c5cbf26acf698f8977" + ], + "linux_ppc64": [ + "go1.22.0.linux-ppc64.tar.gz", + "5ae23bf460731eb078f5700b01a50a303308b9f7174a2994892e28bf061f7c85" + ], + "linux_ppc64le": [ + "go1.22.0.linux-ppc64le.tar.gz", + "0e57f421df9449066f00155ce98a5be93744b3d81b00ee4c2c9b511be2a31d93" + ], + "linux_riscv64": [ + "go1.22.0.linux-riscv64.tar.gz", + "afe9cedcdbd6fdff27c57efd30aa5ce0f666f471fed5fa96cd4fb38d6b577086" + ], + "linux_s390x": [ + "go1.22.0.linux-s390x.tar.gz", + "2e546a3583ba7bd3988f8f476245698f6a93dfa9fe206a8ca8f85c1ceecb2446" + ], + "netbsd_386": [ + "go1.22.0.netbsd-386.tar.gz", + "9b7e2dbd43a169bff18bf533a6c2f462eebe04126ab01c138d0d669c046e1658" + ], + "netbsd_amd64": [ + "go1.22.0.netbsd-amd64.tar.gz", + "b11995c271d2256dfe85cf54882ca3655e18c49c4d7db0502bff9977767894e7" + ], + "netbsd_arm64": [ + "go1.22.0.netbsd-arm64.tar.gz", + "499918ccfedde78264d194244d251bf41d95cf606cc0adad13b36b8103cb974f" + ], + "netbsd_armv6l": [ + "go1.22.0.netbsd-arm.tar.gz", + "b57a3aa9c862300ec0ee8609ce5e0f430f132044f351677fd34711a504081872" + ], + "openbsd_386": [ + "go1.22.0.openbsd-386.tar.gz", + "379e0829104c18a139d78b33378e6bd3ab2e0792f6c43b4c38e7f20d8d77b99d" + ], + "openbsd_amd64": [ + "go1.22.0.openbsd-amd64.tar.gz", + "ceb0c97ffc3bfaf74e1df843cb8571d7fc3173a08432f0f42112495df6a31520" + ], + "openbsd_arm64": [ + "go1.22.0.openbsd-arm64.tar.gz", + "358801cab7122ab50d7d92727644f26d818e9e973403f09e85c2e935a625db75" + ], + "openbsd_armv6l": [ + "go1.22.0.openbsd-arm.tar.gz", + "8af5aea3df539bc95ed412c0a176fe84baced70ea1dd29f4aa82d0e9ce27fd9f" + ], + "plan9_386": [ + "go1.22.0.plan9-386.tar.gz", + "fa42c545c9025c45ca9af176dc13a0f4af0cc26bacff6fcb35bb4a170ac538e8" + ], + "plan9_amd64": [ + "go1.22.0.plan9-amd64.tar.gz", + "d8cf64f37a1dfd8e190c5a303c43ab2d49324868f098d88a3106072d137a5a0b" + ], + "plan9_armv6l": [ + "go1.22.0.plan9-arm.tar.gz", + "86fd6165f0cbb47ad551094f74b3e5a6c5e09de858d8b99de72d978d41be6e2a" + ], + "solaris_amd64": [ + "go1.22.0.solaris-amd64.tar.gz", + "a6c12651768d3a74f16104502b4b7bef513ea6b646d99990a28d934c261d1689" + ], + "windows_386": [ + "go1.22.0.windows-386.zip", + "553d44928509965cbda02a45b35ab01cf8b925534bc526a34e2d9dc7794b57e8" + ], + "windows_amd64": [ + "go1.22.0.windows-amd64.zip", + "78b3158fe3aa358e0b6c9f26ecd338f9a11441e88bc434ae2e9f0ca2b0cc4dd3" + ], + "windows_arm64": [ + "go1.22.0.windows-arm64.zip", + "31a61e41d06a3bb2189a303f5f3e777ca4b454eff439f0a67bc2b166330021f4" + ], + "windows_armv6l": [ + "go1.22.0.windows-arm.zip", + "495c7dfaea4e2bf48643662bb622e4ce6378d6d9840015238ad4b8792b99ddbf" + ] + }, + "1.22.4": { + "aix_ppc64": [ + "go1.22.4.aix-ppc64.tar.gz", + "b9647fa9fc83a0cc5d4f092a19eaeaecf45f063a5aa7d4962fde65aeb7ae6ce1" + ], + "darwin_amd64": [ + "go1.22.4.darwin-amd64.tar.gz", + "c95967f50aa4ace34af0c236cbdb49a9a3e80ee2ad09d85775cb4462a5c19ed3" + ], + "darwin_arm64": [ + "go1.22.4.darwin-arm64.tar.gz", + "242b78dc4c8f3d5435d28a0d2cec9b4c1aa999b601fb8aa59fb4e5a1364bf827" + ], + "dragonfly_amd64": [ + "go1.22.4.dragonfly-amd64.tar.gz", + "f2fbb51af4719d3616efb482d6ed2b96579b474156f85a7ddc6f126764feec4b" + ], + "freebsd_386": [ + "go1.22.4.freebsd-386.tar.gz", + "7c54884bb9f274884651d41e61d1bc12738863ad1497e97ea19ad0e9aa6bf7b5" + ], + "freebsd_amd64": [ + "go1.22.4.freebsd-amd64.tar.gz", + "88d44500e1701dd35797619774d6dd51bf60f45a8338b0a82ddc018e4e63fb78" + ], + "freebsd_arm64": [ + "go1.22.4.freebsd-arm64.tar.gz", + "726dc093cf020277be45debf03c3b02b43c2efb3e2a5d4fba8f52579d65327dc" + ], + "freebsd_armv6l": [ + "go1.22.4.freebsd-arm.tar.gz", + "3d9efe47db142a22679aba46b1772e3900b0d87ae13bd2b3bc80dbf2ac0b2cd6" + ], + "freebsd_riscv64": [ + "go1.22.4.freebsd-riscv64.tar.gz", + "5f6b67e5e32f1d6ccb2d4dcb44934a5e2e870a877ba7443d86ec43cfc28afa71" + ], + "illumos_amd64": [ + "go1.22.4.illumos-amd64.tar.gz", + "d56ecc2f85b6418a21ef83879594d0c42ab4f65391a676bb12254870e6690d63" + ], + "linux_386": [ + "go1.22.4.linux-386.tar.gz", + "47a2a8d249a91eb8605c33bceec63aedda0441a43eac47b4721e3975ff916cec" + ], + "linux_amd64": [ + "go1.22.4.linux-amd64.tar.gz", + "ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d" + ], + "linux_arm64": [ + "go1.22.4.linux-arm64.tar.gz", + "a8e177c354d2e4a1b61020aca3562e27ea3e8f8247eca3170e3fa1e0c2f9e771" + ], + "linux_armv6l": [ + "go1.22.4.linux-armv6l.tar.gz", + "e2b143fbacbc9cbd448e9ef41ac3981f0488ce849af1cf37e2341d09670661de" + ], + "linux_loong64": [ + "go1.22.4.linux-loong64.tar.gz", + "e2ff9436e4b34bf6926b06d97916e26d67a909a2effec17967245900f0816f1d" + ], + "linux_mips": [ + "go1.22.4.linux-mips.tar.gz", + "73f0dcc60458c4770593b05a7bc01cc0d31fc98f948c0c2334812c7a1f2fc3f1" + ], + "linux_mips64": [ + "go1.22.4.linux-mips64.tar.gz", + "417af97fc2630a647052375768be4c38adcc5af946352ea5b28613ea81ca5d45" + ], + "linux_mips64le": [ + "go1.22.4.linux-mips64le.tar.gz", + "7486e2d7dd8c98eb44df815ace35a7fe7f30b7c02326e3741bd934077508139b" + ], + "linux_mipsle": [ + "go1.22.4.linux-mipsle.tar.gz", + "69479c8aad301e459a8365b40cad1074a0dbba5defb9291669f94809c4c4be6e" + ], + "linux_ppc64": [ + "go1.22.4.linux-ppc64.tar.gz", + "dd238847e65bc3e2745caca475a5db6522a2fcf85cf6c38fc36a06642b19efd7" + ], + "linux_ppc64le": [ + "go1.22.4.linux-ppc64le.tar.gz", + "a3e5834657ef92523f570f798fed42f1f87bc18222a16815ec76b84169649ec4" + ], + "linux_riscv64": [ + "go1.22.4.linux-riscv64.tar.gz", + "56a827ff7dc6245bcd7a1e9288dffaa1d8b0fd7468562264c1523daf3b4f1b4a" + ], + "linux_s390x": [ + "go1.22.4.linux-s390x.tar.gz", + "7590c3e278e2dc6040aae0a39da3ca1eb2e3921673a7304cc34d588c45889eec" + ], + "netbsd_386": [ + "go1.22.4.netbsd-386.tar.gz", + "ddd2eebe34471a2502de6c5dad04ab27c9fc80cbde7a9ad5b3c66ecec4504e1d" + ], + "netbsd_amd64": [ + "go1.22.4.netbsd-amd64.tar.gz", + "33af79f6f935f6fbacc5d23876450b3567b79348fc065beef8e64081127dd234" + ], + "netbsd_arm64": [ + "go1.22.4.netbsd-arm64.tar.gz", + "c9a2971dec9f6d320c6f2b049b2353c6d0a2d35e87b8a4b2d78a2f0d62545f8e" + ], + "netbsd_armv6l": [ + "go1.22.4.netbsd-arm.tar.gz", + "fa3550ebd5375a70b3bcd342b5a71f4bd271dcbbfaf4eabefa2144ab5d8924b6" + ], + "openbsd_386": [ + "go1.22.4.openbsd-386.tar.gz", + "d21af022331bfdc2b5b161d616c3a1a4573d33cf7a30416ee509a8f3641deb47" + ], + "openbsd_amd64": [ + "go1.22.4.openbsd-amd64.tar.gz", + "72c0094c43f7e5722ec49c2a3e9dfa7a1123ac43a5f3a63eecf3e3795d3ff0ae" + ], + "openbsd_arm64": [ + "go1.22.4.openbsd-arm64.tar.gz", + "a7ab8d4e0b02bf06ed144ba42c61c0e93ee00f2b433415dfd4ad4b6e79f31650" + ], + "openbsd_armv6l": [ + "go1.22.4.openbsd-arm.tar.gz", + "1096831ea3c5ea3ca57d14251d9eda3786889531eb40d7d6775dcaa324d4b065" + ], + "openbsd_ppc64": [ + "go1.22.4.openbsd-ppc64.tar.gz", + "9716327c8a628358798898dc5148c49dbbeb5196bf2cbf088e550721a6e4f60b" + ], + "plan9_386": [ + "go1.22.4.plan9-386.tar.gz", + "a8dd4503c95c32a502a616ab78870a19889c9325fe9bd31eb16dd69346e4bfa8" + ], + "plan9_amd64": [ + "go1.22.4.plan9-amd64.tar.gz", + "5423a25808d76fe5aca8607a2e5ac5673abf45446b168cb5e9d8519ee9fe39a1" + ], + "plan9_armv6l": [ + "go1.22.4.plan9-arm.tar.gz", + "6af939ad583f5c85c09c53728ab7d38c3cc2b39167562d6c18a07c5c6608b370" + ], + "solaris_amd64": [ + "go1.22.4.solaris-amd64.tar.gz", + "e8cabe69c03085725afdb32a6f9998191a3e55a747b270d835fd05000d56abba" + ], + "windows_386": [ + "go1.22.4.windows-386.zip", + "aca4e2c37278a10f1c70dd0df142f7d66b50334fcee48978d409202d308d6d25" + ], + "windows_amd64": [ + "go1.22.4.windows-amd64.zip", + "26321c4d945a0035d8a5bc4a1965b0df401ff8ceac66ce2daadabf9030419a98" + ], + "windows_arm64": [ + "go1.22.4.windows-arm64.zip", + "8a2daa9ea28cbdafddc6171aefed384f4e5b6e714fb52116fe9ed25a132f37ed" + ], + "windows_armv6l": [ + "go1.22.4.windows-arm.zip", + "5fcd0671a49cecf39b41021621ee1b6e7aa1370f37122b72e80d4fd4185833b6" + ] + }, + "1.24.6": { + "aix_ppc64": [ + "go1.24.6.aix-ppc64.tar.gz", + "5cafb2927af95969fc3d0884be1ad4df54f19103d86dda928d9f76494a5ae6a6" + ], + "darwin_amd64": [ + "go1.24.6.darwin-amd64.tar.gz", + "4a8d7a32052f223e71faab424a69430455b27b3fff5f4e651f9d97c3e51a8746" + ], + "darwin_arm64": [ + "go1.24.6.darwin-arm64.tar.gz", + "4e29202c49573b953be7cc3500e1f8d9e66ddd12faa8cf0939a4951411e09a2a" + ], + "dragonfly_amd64": [ + "go1.24.6.dragonfly-amd64.tar.gz", + "c6e21bf8347f7a1c653dd4136c8c27a858515e98501d2843023a4bb3f1f7fb63" + ], + "freebsd_386": [ + "go1.24.6.freebsd-386.tar.gz", + "9cd74ad74f3ad833e92529f2fd9b0d7d9ffaab46307eccadb0afcf9a1ba09553" + ], + "freebsd_amd64": [ + "go1.24.6.freebsd-amd64.tar.gz", + "4983e2b10ae1f754e4eb07e1e589691c7e1d0dc428a92c16bd0e2ba03cc23ed9" + ], + "freebsd_arm": [ + "go1.24.6.freebsd-arm.tar.gz", + "a8da621d8282a91ee17b257a46f2606391c019cc1a7d7be628638792ca8033ad" + ], + "freebsd_arm64": [ + "go1.24.6.freebsd-arm64.tar.gz", + "76a75ad5125217c268029c0ad9c7295cc7f6042fe9cba4bebf9a89f7f42ad8af" + ], + "freebsd_riscv64": [ + "go1.24.6.freebsd-riscv64.tar.gz", + "ac206417d8460662f26d46dc2ad0488b2f9e22039946069ba4b48a0cb646e8b0" + ], + "illumos_amd64": [ + "go1.24.6.illumos-amd64.tar.gz", + "c54199c46ec823c857c52335f859b493593433d71fe479b9e7f95d868a144ae2" + ], + "linux_386": [ + "go1.24.6.linux-386.tar.gz", + "bb5bf69d75e7edbc93339824753a1a4655a928451a2c5e13ff90959ad69e065b" + ], + "linux_amd64": [ + "go1.24.6.linux-amd64.tar.gz", + "bbca37cc395c974ffa4893ee35819ad23ebb27426df87af92e93a9ec66ef8712" + ], + "linux_arm64": [ + "go1.24.6.linux-arm64.tar.gz", + "124ea6033a8bf98aa9fbab53e58d134905262d45a022af3a90b73320f3c3afd5" + ], + "linux_armv6l": [ + "go1.24.6.linux-armv6l.tar.gz", + "7feb4d25f5e72f94fda81c99d4adb6630dfa2c35211e0819417d53af6e71809e" + ], + "linux_loong64": [ + "go1.24.6.linux-loong64.tar.gz", + "8424d9c52b254255d2720769366af73802747e4eaf339c6e86bb52e5a1809fcb" + ], + "linux_mips": [ + "go1.24.6.linux-mips.tar.gz", + "791964a6ce65f604a2099b7e61f71128cc7a3a4fcee858218d2ce99033fc2b09" + ], + "linux_mips64": [ + "go1.24.6.linux-mips64.tar.gz", + "2f5282fdf2eaca44515728d518caec888b0dc4948effecb8e7f2ddff1ff9aa9e" + ], + "linux_mips64le": [ + "go1.24.6.linux-mips64le.tar.gz", + "64ddc6b28907b4ab997f2d81c226be403511a59a6100536560af26ccfb10b4a4" + ], + "linux_mipsle": [ + "go1.24.6.linux-mipsle.tar.gz", + "5ca12d2d4939dea977100e76885f4e08cd3ef3b73823853527b58ecac4598fd3" + ], + "linux_ppc64": [ + "go1.24.6.linux-ppc64.tar.gz", + "9a4a61ffbba7a58b3ef97a6d3b2b394168f613f9ca6092016981a88c78a57f86" + ], + "linux_ppc64le": [ + "go1.24.6.linux-ppc64le.tar.gz", + "63fc9559a3d6dfd63aa902f714375b879bbc848466181c035c122489b9646e27" + ], + "linux_riscv64": [ + "go1.24.6.linux-riscv64.tar.gz", + "e92c19ff15a9004fe43e1f62d433555250ef9fe3fabee3dc29ebd4802c2b8021" + ], + "linux_s390x": [ + "go1.24.6.linux-s390x.tar.gz", + "4cde28d9ffb6eef86bf8dac6852a45db335009f67e60bee3e477dd0ba0ff9704" + ], + "netbsd_386": [ + "go1.24.6.netbsd-386.tar.gz", + "d238045676104d2aaad9da53c4ba3f489cf81b26d3315d2caad2b6e240664423" + ], + "netbsd_amd64": [ + "go1.24.6.netbsd-amd64.tar.gz", + "9f8a9245cf146f1ebdac6785f71646c3dd0edf07dcdb8e5a8df85ec6447e5308" + ], + "netbsd_arm": [ + "go1.24.6.netbsd-arm.tar.gz", + "2359c8af6b2d6979910060478a57fd216824682e0908ea1a4412ea8dc4a55db7" + ], + "netbsd_arm64": [ + "go1.24.6.netbsd-arm64.tar.gz", + "99101158c796997ec85023d395c133fa90c782bd5faa56f8c3824817d28b0ee8" + ], + "openbsd_386": [ + "go1.24.6.openbsd-386.tar.gz", + "b6193420489f24d5560b87ccb38caf52cecb308d892d99bf21d1ef0d2c99de04" + ], + "openbsd_amd64": [ + "go1.24.6.openbsd-amd64.tar.gz", + "8dc0d94afbc65be9cafebe543b88a841dcf3429e50f668ea714200ef9f4489bd" + ], + "openbsd_arm": [ + "go1.24.6.openbsd-arm.tar.gz", + "c29563d14d8e4b5fa89f130afac40abba7a1ee1e87fcc759a95f1e9b5831bf68" + ], + "openbsd_arm64": [ + "go1.24.6.openbsd-arm64.tar.gz", + "3d97bdc5e1997cccfaa398242c52e8a97d3b626d988a65248bbdbf2f1ab188bf" + ], + "openbsd_ppc64": [ + "go1.24.6.openbsd-ppc64.tar.gz", + "54cf517933db0bff58f3e527e7fa194f505d562c71b1fb0050d3365fcdda4914" + ], + "openbsd_riscv64": [ + "go1.24.6.openbsd-riscv64.tar.gz", + "f7bdd8f882ecbc01ff6c1782ce3e29298445dd99b3dd4299423257df307c7b0d" + ], + "plan9_386": [ + "go1.24.6.plan9-386.tar.gz", + "9bc0a801dc0d8d0705ac9493219b926b2e3c0105b8921c6c2fce69d14c4b9007" + ], + "plan9_amd64": [ + "go1.24.6.plan9-amd64.tar.gz", + "7133c28acc277d3c5e8317fd9454b3a0902138c384466f818644b504b8b89ffe" + ], + "plan9_arm": [ + "go1.24.6.plan9-arm.tar.gz", + "c4e2aa8aa4353f75fc083ca728b73917467cef88c6f5f917ca420578672c6d58" + ], + "solaris_amd64": [ + "go1.24.6.solaris-amd64.tar.gz", + "cfcb4f1f7e987c6dee893c4401546516cc271c1ac7d7fb37a32b25343dc7df32" + ], + "windows_386": [ + "go1.24.6.windows-386.zip", + "39b4d31b933d2f7c8913e70fb0fffef27252e441c386eac5c13632cbb28dfb6e" + ], + "windows_amd64": [ + "go1.24.6.windows-amd64.zip", + "4fbc8af2cfca9e5059019b5150a426eb78e1e57718bf08f0e52b1c942a2782bf" + ], + "windows_arm64": [ + "go1.24.6.windows-arm64.zip", + "45c41b237d00e92e4cf8adce11b4c5258048b47a92bfbb1f4ef3b928d6fcb0b2" + ] + }, + "1.25.0": { + "aix_ppc64": [ + "go1.25.0.aix-ppc64.tar.gz", + "e5234a7dac67bc86c528fe9752fc9d63557918627707a733ab4cac1a6faed2d4" + ], + "darwin_amd64": [ + "go1.25.0.darwin-amd64.tar.gz", + "5bd60e823037062c2307c71e8111809865116714d6f6b410597cf5075dfd80ef" + ], + "darwin_arm64": [ + "go1.25.0.darwin-arm64.tar.gz", + "544932844156d8172f7a28f77f2ac9c15a23046698b6243f633b0a0b00c0749c" + ], + "dragonfly_amd64": [ + "go1.25.0.dragonfly-amd64.tar.gz", + "5ed3cf9a810a1483822538674f1336c06b51aa1b94d6d545a1a0319a48177120" + ], + "freebsd_386": [ + "go1.25.0.freebsd-386.tar.gz", + "abea5d5c6697e6b5c224731f2158fe87c602996a2a233ac0c4730cd57bf8374e" + ], + "freebsd_amd64": [ + "go1.25.0.freebsd-amd64.tar.gz", + "86e6fe0a29698d7601c4442052dac48bd58d532c51cccb8f1917df648138730b" + ], + "freebsd_arm": [ + "go1.25.0.freebsd-arm.tar.gz", + "d90b78e41921f72f30e8bbc81d9dec2cff7ff384a33d8d8debb24053e4336bfe" + ], + "freebsd_arm64": [ + "go1.25.0.freebsd-arm64.tar.gz", + "451d0da1affd886bfb291b7c63a6018527b269505db21ce6e14724f22ab0662e" + ], + "freebsd_riscv64": [ + "go1.25.0.freebsd-riscv64.tar.gz", + "7b565f76bd8bda46549eeaaefe0e53b251e644c230577290c0f66b1ecdb3cdbe" + ], + "illumos_amd64": [ + "go1.25.0.illumos-amd64.tar.gz", + "b1e1fdaab1ad25aa1c08d7a36c97d45d74b98b89c3f78c6d2145f77face54a2c" + ], + "linux_386": [ + "go1.25.0.linux-386.tar.gz", + "8c602dd9d99bc9453b3995d20ce4baf382cc50855900a0ece5de9929df4a993a" + ], + "linux_amd64": [ + "go1.25.0.linux-amd64.tar.gz", + "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613" + ], + "linux_arm64": [ + "go1.25.0.linux-arm64.tar.gz", + "05de75d6994a2783699815ee553bd5a9327d8b79991de36e38b66862782f54ae" + ], + "linux_armv6l": [ + "go1.25.0.linux-armv6l.tar.gz", + "a5a8f8198fcf00e1e485b8ecef9ee020778bf32a408a4e8873371bfce458cd09" + ], + "linux_loong64": [ + "go1.25.0.linux-loong64.tar.gz", + "cab86b1cf761b1cb3bac86a8877cfc92e7b036fc0d3084123d77013d61432afc" + ], + "linux_mips": [ + "go1.25.0.linux-mips.tar.gz", + "d66b6fb74c3d91b9829dc95ec10ca1f047ef5e89332152f92e136cf0e2da5be1" + ], + "linux_mips64": [ + "go1.25.0.linux-mips64.tar.gz", + "4082e4381a8661bc2a839ff94ba3daf4f6cde20f8fb771b5b3d4762dc84198a2" + ], + "linux_mips64le": [ + "go1.25.0.linux-mips64le.tar.gz", + "70002c299ec7f7175ac2ef673b1b347eecfa54ae11f34416a6053c17f855afcc" + ], + "linux_mipsle": [ + "go1.25.0.linux-mipsle.tar.gz", + "b00a3a39eff099f6df9f1c7355bf28e4589d0586f42d7d4a394efb763d145a73" + ], + "linux_ppc64": [ + "go1.25.0.linux-ppc64.tar.gz", + "df166f33bd98160662560a72ff0b4ba731f969a80f088922bddcf566a88c1ec1" + ], + "linux_ppc64le": [ + "go1.25.0.linux-ppc64le.tar.gz", + "0f18a89e7576cf2c5fa0b487a1635d9bcbf843df5f110e9982c64df52a983ad0" + ], + "linux_riscv64": [ + "go1.25.0.linux-riscv64.tar.gz", + "c018ff74a2c48d55c8ca9b07c8e24163558ffec8bea08b326d6336905d956b67" + ], + "linux_s390x": [ + "go1.25.0.linux-s390x.tar.gz", + "34e5a2e19f2292fbaf8783e3a241e6e49689276aef6510a8060ea5ef54eee408" + ], + "netbsd_386": [ + "go1.25.0.netbsd-386.tar.gz", + "f8586cdb7aa855657609a5c5f6dbf523efa00c2bbd7c76d3936bec80aa6c0aba" + ], + "netbsd_amd64": [ + "go1.25.0.netbsd-amd64.tar.gz", + "ae8dc1469385b86a157a423bb56304ba45730de8a897615874f57dd096db2c2a" + ], + "netbsd_arm": [ + "go1.25.0.netbsd-arm.tar.gz", + "1ff7e4cc764425fc9dd6825eaee79d02b3c7cafffbb3691687c8d672ade76cb7" + ], + "netbsd_arm64": [ + "go1.25.0.netbsd-arm64.tar.gz", + "e1b310739f26724216aa6d7d7208c4031f9ff54c9b5b9a796ddc8bebcb4a5f16" + ], + "openbsd_386": [ + "go1.25.0.openbsd-386.tar.gz", + "4802a9b20e533da91adb84aab42e94aa56cfe3e5475d0550bed3385b182e69d8" + ], + "openbsd_amd64": [ + "go1.25.0.openbsd-amd64.tar.gz", + "c016cd984bebe317b19a4f297c4f50def120dc9788490540c89f28e42f1dabe1" + ], + "openbsd_arm": [ + "go1.25.0.openbsd-arm.tar.gz", + "a1e31d0bf22172ddde42edf5ec811ef81be43433df0948ece52fecb247ccfd8d" + ], + "openbsd_arm64": [ + "go1.25.0.openbsd-arm64.tar.gz", + "343ea8edd8c218196e15a859c6072d0dd3246fbbb168481ab665eb4c4140458d" + ], + "openbsd_ppc64": [ + "go1.25.0.openbsd-ppc64.tar.gz", + "694c14da1bcaeb5e3332d49bdc2b6d155067648f8fe1540c5de8f3cf8e157154" + ], + "openbsd_riscv64": [ + "go1.25.0.openbsd-riscv64.tar.gz", + "aa510ad25cf54c06cd9c70b6d80ded69cb20188ac6e1735655eef29ff7e7885f" + ], + "plan9_386": [ + "go1.25.0.plan9-386.tar.gz", + "46f8cef02086cf04bf186c5912776b56535178d4cb319cd19c9fdbdd29231986" + ], + "plan9_amd64": [ + "go1.25.0.plan9-amd64.tar.gz", + "29b34391d84095e44608a228f63f2f88113a37b74a79781353ec043dfbcb427b" + ], + "plan9_arm": [ + "go1.25.0.plan9-arm.tar.gz", + "0a047107d13ebe7943aaa6d54b1d7bbd2e45e68ce449b52915a818da715799c2" + ], + "solaris_amd64": [ + "go1.25.0.solaris-amd64.tar.gz", + "9977f9e4351984364a3b2b78f8b88bfd1d339812356d5237678514594b7d3611" + ], + "windows_386": [ + "go1.25.0.windows-386.zip", + "df9f39db82a803af0db639e3613a36681ab7a42866b1384b3f3a1045663961a7" + ], + "windows_amd64": [ + "go1.25.0.windows-amd64.zip", + "89efb4f9b30812eee083cc1770fdd2913c14d301064f6454851428f9707d190b" + ], + "windows_arm64": [ + "go1.25.0.windows-arm64.zip", + "27bab004c72b3d7bd05a69b6ec0fc54a309b4b78cc569dd963d8b3ec28bfdb8c" + ] + } } - } + }, + "factsVersions": {} } diff --git a/README.md b/README.md index b511fc1d..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -2,197 +2,139 @@ Website | Getting Started - | Documentation + | Documentation | Discord | Contributing [ZML]: https://zml.ai/ [Getting Started]: #getting-started -[Documentation]: https://docs.zml.ai +[Documentation]: ./docs/README.md [Contributing]: ./CONTRIBUTING.md [Discord]: https://discord.gg/6y72SN2E7H -# Bonjour 👋 +# About -At ZML, we are creating exciting AI products on top of our high-performance -AI inference stack. Our stack is built for production, using the amazing -[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and the -power of [Bazel](https://bazel.build). +ZML is a production inference stack, purpose-built to decouple AI workloads from proprietary hardware. -
-
Take me straight to getting started or give me a taste 🥐!
-
+Any model, many hardwares, one codebase, peak performance. ---- - -  - -# We're happy to share! -We're very happy to share our inference stack with the World and hope it allows -you, too, to build cool and exciting AI projects. - -To give you a glimpse of what you can do with ZML, here is an early demo: - -
- -It shows a prototype running a LLama3 model sharded on 1 NVIDIA RTX 4090, 1 AMD -6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different -locations, with activations being passed over a VPN. - -All processes used the same model code, cross-compiled on a Mac, and copied onto -the servers. - -For more inspiration, see also the examples below or check out the -[examples](./examples) folder. - - - -# Getting started +Compiled directly to NVIDIA, AMD, INTEL, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. +It is built using the +[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). +# Getting Started ## Prerequisites We use `bazel` to build ZML and its dependencies. The only prerequisite is -`bazel`, which we recommend to download through `bazelisk`, a version manager -for `bazel`. - -**Please note: If you do not wish to install `bazel`** system-wide, we provide -[bazel.sh](bazel.sh) which downloads it to your home folder -and runs it. - -**Install Bazel** (recommended): - -
+`bazel`, which we recommend installing through `bazelisk`. ### macOS - -``` +```bash brew install bazelisk ``` -
- -
### Linux - -``` -curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' +```bash +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.0/bazelisk-linux-amd64' chmod +x /usr/local/bin/bazel ``` -
+## 30-Second Smoke Test -## Run a pre-packaged model +Run the MNIST example: -We have implemented a variety of example models in ZML. See our reference -implementations in the -[examples](https://github.com/zml/zml/tree/master/examples/) folder. - -### MNIST - -The [classic](https://en.wikipedia.org/wiki/MNIST_database) handwritten digits -recognition task. The model is tasked to recognize a handwritten digit, which -has been converted to a 28x28 pixel monochrome image. `Bazel` will download a -pre-trained model, and the test dataset. The program will load the model, -compile it, and classify a randomly picked example from the test dataset. - -On the command line: - -``` -bazel run --config=release //examples/mnist - -# or -./bazel.sh run --config=release //examples/mnist +```bash +bazel run //examples/mnist ``` +This downloads a small pretrained MNIST model, compiles it, loads the weights, and +classifies a random handwritten digit. +## LLM Quickstart -### Meta Llama 3.1 8B +The main LLM example is [`//examples/llm`](./examples/llm). It currently supports: -This model has restrictions, see -[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). It **requires -approval from Meta on Huggingface**, which can take a few hours to get granted. +- Llama 3.1 / 3.2 +- Qwen 3.5 +- LFM 2.5 -Once you've been granted access, you're ready to download a gated model like -`Llama-3.1-8B-Instruct`! +Authenticate with Hugging Face if you want to load gated repos such as Meta +Llama: -First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). -Note you don't need to install it yourself, -you can just use the packaged version `bazel run //tools/hf --`. - -``` -bazel run //tools/hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' +```bash +bazel run //tools/hf -- auth login ``` -Then, you can run the model. +Alternatively, set the `HF_TOKEN` environment variable. -``` -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +Then run a prompt directly: + +```bash +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` -You can also try `Llama-3.1-70B-Instruct` if you have enough memory. +Open the interactive chat loop by omitting `--prompt`: -### Meta Llama 3.2 1B - -Like the 8B model above, this model also requires approval. See -[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. - -``` -bazel run //tools/hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" +```bash +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct ``` -For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. +You can also load from: + +- a local directory: `--model=/var/models/meta-llama/Llama-3.2-1B-Instruct` +- S3: `--model=s3://bucket/path/to/model` ## Running Models on GPU / TPU -You can compile models for accelerator runtimes by appending one or more of the -following arguments to the command line when compiling / running a model: +Append one or more platform flags when compiling or running: - NVIDIA CUDA: `--@zml//platforms:cuda=true` - AMD RoCM: `--@zml//platforms:rocm=true` +- Intel OneAPI: `--@zml//platforms:oneapi=true` - Google TPU: `--@zml//platforms:tpu=true` -- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` -- **AVOID CPU:** `--@zml//platforms:cpu=false` +- AWS Trainium / Inferentia 2: `--@zml//platforms:neuron=true` +- Disable CPU compilation: `--@zml//platforms:cpu=false` -The latter, avoiding compilation for CPU, cuts down compilation time. +Example on CUDA: -So, to run the Llama 3.1 8B model from above on your host sporting an NVIDIA GPU, -run the following: - -``` -bazel run --config=release //examples/llama \ - --@zml//platforms:cuda=true \ - -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ - --prompt="What is the capital of France?" +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" ``` +Example on ROCm: + +```bash +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +``` + +Example on Intel OneAPI: + +```bash +bazel run //examples/llm --@zml//platforms:cpu=false --@zml//platforms:oneapi=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +``` ## Run Tests -``` +```bash bazel test //zml:test ``` +# Examples -# A taste of ZML - - - -## MNIST +- [`examples/llm`](./examples/llm): unified LLM CLI for Llama, Qwen, and LFM +- [`examples/mnist`](./examples/mnist): smallest end-to-end model run +- [`examples/sharding`](./examples/sharding): logical mesh, partitioners, shard-local execution, profiler output +- [`examples/io`](./examples/io): inspect and load local, `hf://`, `https://`, and `s3://` repositories through the VFS layer +- [`examples/benchmark`](./examples/benchmark): measure loading and execution performance +# A Taste Of ZML ```zig -const std = @import("std"); -const zml = @import("zml"); - -/// Model definition const Mnist = struct { fc1: Layer, fc2: Layer, @@ -201,65 +143,84 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, + pub fn init(store: zml.io.TensorStore.View) Layer { + return .{ + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), + }; + } + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + .fc1 = .init(store.withPrefix("fc1")), + .fc2 = .init(store.withPrefix("fc2")), + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + meshes: []const *const zml.Mesh, + ) !zml.Bufferized(Mnist) { + return zml.io.load(Mnist, self, allocator, io, platform, store, .{ + .meshes = meshes, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = zml.call(layer, .forward, .{x}); + x = layer.forward(x); } - return x.argMax(0, .u8).indices; + return x.argMax(0).indices.convert(.u8); } }; ``` +For a full walkthrough, see: +- [Getting Started](./docs/tutorials/getting_started.md) +- [Writing your first model](./docs/tutorials/write_first_model.md) +- [ZML Concepts](./docs/learn/concepts.md) +- [Deploying on a server](./docs/howtos/deploy_on_server.md) -## Tagged Tensors - -```zig -const Sdpa = struct { - pub fn forward(_: Sdpa, ctx: *zml.Context, q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor) zml.Tensor { - const q = q_.withTags(.{ .b, .h, .q, .hd }); - const k = k_.withTags(.{ .b, .h, .k, .hd }); - const v = v_.withTags(.{ .b, .h, .k, .hd }); - const attn_mask = zml.nn.causalAttnMask(ctx, .{ .q = q.dim(.q), .k = k.dim(.k) }, q.dtype(), null); - return zml.nn.sdpa(ctx, q, k, v, .{ .attn_mask = attn_mask }); - } -}; -``` - - - - -# Where to go next: - -You might want to check out more [examples](./examples), read through the -[documentation directly on GitHub](./docs/README.md), or, for the full rendering -experience, browse the -[online documentation with included API reference](https://docs.zml.ai). - +# Where To Go Next +- Run more examples in [`./examples`](./examples) +- Read the example-specific notes in [`examples/llm/README.md`](./examples/llm/README.md) +- Learn tagged dimensions in [`working_with_tensors.md`](./docs/tutorials/working_with_tensors.md) +- Start building a model with [`write_first_model.md`](./docs/tutorials/write_first_model.md) +- Explore deployment in [`deploy_on_server.md`](./docs/howtos/deploy_on_server.md) # Contributing See [here][Contributing]. - - # License ZML is licensed under the [Apache 2.0 license](./LICENSE). - - -# Thanks to our contributors +# Thanks To Our Contributors diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index bb4a1781..eb2b2ab2 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -1,10 +1,24 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") +cc_library( + name = "retain_runfiles_init", + srcs = ["runfiles_init.c"], + # Retain the constructor even when its object is in a static archive. + # Mach-O adds an underscore to the exported C symbol. + linkopts = select({ + "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], + "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], + }), +) + zig_library( name = "bazel", main = "bazel.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ + ":retain_runfiles_init", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 6d5540b0..67513227 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,52 +1,27 @@ const std = @import("std"); -const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _ linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; // autofix - _ = envp; // autofix - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIRECTORY")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; - -pub fn init(allocator: std.mem.Allocator, io: std.Io, argv: std.process.Args, environ_map: *std.process.Environ.Map) !void { - runfiles_global = try allocator.create(bazel_runfiles.Runfiles); - runfiles_global.* = try bazel_runfiles.Runfiles.create(.{ - .allocator = allocator, - .io = io, - .argv = argv, - .environ_map = environ_map, - }) orelse { +pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); }; } -pub fn initSimple(init_: std.process.Init) !void { - try init(init_.arena.allocator(), init_.io, init_.minimal.args, init_.environ_map); -} - pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); } diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index 7c0e8de5..fc6ef727 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -1,4 +1,5 @@ -load("@bazel_tools//tools/build_defs/cc:cc_import.bzl", _cc_import = "cc_import") +load("@rules_cc//cc:defs.bzl", "cc_library", _cc_import = "cc_import") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":patchelf.bzl", "patchelf") def _cc_import_runfiles_impl(ctx): @@ -70,7 +71,7 @@ def cc_import( interface_library = interface_library, visibility = visibility, ) - native.cc_library( + cc_library( name = name, data = data, deps = deps + [name + ".cc_import"], diff --git a/bazel/http_aea_archive.bzl b/bazel/http_aea_archive.bzl new file mode 100644 index 00000000..439a6a41 --- /dev/null +++ b/bazel/http_aea_archive.bzl @@ -0,0 +1,164 @@ +"""Repository rule for downloading and extracting Apple Encrypted Archive assets on macOS and Linux AMD64/ARM64.""" + +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") + +def _fail_result(step, result): + if result.return_code != 0: + fail("%s failed with exit code %d\nstdout:\n%s\nstderr:\n%s" % ( + step, + result.return_code, + result.stdout, + result.stderr, + )) + +def _execute(rctx, step, argv, quiet = True): + result = rctx.execute([str(arg) for arg in argv], quiet = quiet) + _fail_result(step, result) + return result + +def _host_tools(rctx): + platform = repo_utils.platform(rctx) + if platform not in ["darwin_amd64", "darwin_arm64", "linux_amd64", "linux_arm64"]: + fail("http_aea_archive supports macOS and Linux AMD64/ARM64 repository hosts; got %s/%s" % (rctx.os.name, rctx.os.arch)) + ipsw = Label("@ipsw_{}//:ipsw".format(platform)) if platform.startswith("linux_") else None + sevenzip = Label("@sevenzip_{}//:7zz".format(platform)) + return (ipsw, sevenzip) + +def _single_restore_dmg(restore_dirs): + dmgs = [] + entries = [] + for restore_dir in restore_dirs: + if not restore_dir.exists: + continue + for entry in restore_dir.readdir(): + entries.append(str(entry)) + if not entry.is_dir and entry.basename.endswith(".dmg"): + dmgs.append(entry) + + if len(dmgs) != 1: + fail("expected exactly one DMG under AssetData/Restore, got %d; entries: %s" % ( + len(dmgs), + ", ".join(sorted(entries)), + )) + + return dmgs[0] + +def _linux_restore_dirs(rctx, ipsw, aar): + # Create an explicit empty config. It avoids loading the user's ipsw configuration. + rctx.file("ipsw-config.yaml", "{}\n") + _execute(rctx, "ipsw ota extract", [ + rctx.path(ipsw), + "--config", + rctx.path("ipsw-config.yaml"), + "ota", + "extract", + aar, + "--key-val", + rctx.attr.archive_decryption_key, + "--pattern", + "AssetData/Restore/.*\\.dmg$", # pattern to match DMG files under AssetData/Restore + "--output", + rctx.path("ipsw-output"), + "--confirm", + "--no-color", + ]) + output = rctx.path("ipsw-output") + restore_dirs = [] + if output.exists: + # ipsw prefixes extracted paths with an asset-specific directory. + for entry in output.readdir(): + if entry.is_dir: + restore_dirs.append(entry.get_child("AssetData", "Restore")) + return restore_dirs + +def _http_aea_archive_impl(rctx): + ipsw, sevenzip = _host_tools(rctx) + + urls = [] + if rctx.attr.url: + urls.append(rctx.attr.url) + urls.extend(rctx.attr.urls) + if not urls: + fail("http_aea_archive requires url or urls") + + if not rctx.attr.sha256 and not rctx.attr.integrity: + fail("http_aea_archive requires sha256 or integrity for reproducible downloads") + if not rctx.attr.archive_decryption_key: + fail("archive_decryption_key is required") + + aar = rctx.path("asset.aar") # path to the downloaded Apple Encrypted Archive (AAR) file + download_kwargs = { + "canonical_id": rctx.attr.canonical_id, + "integrity": rctx.attr.integrity, + "output": aar, + "sha256": rctx.attr.sha256, + "url": urls, + } + if rctx.attr.netrc: + download_kwargs["auth"] = rctx.use_netrc( + rctx.read_netrc(rctx.attr.netrc), + urls, + rctx.attr.auth_patterns, + ) + + rctx.download(**download_kwargs) + + if ipsw: + restore_dirs = _linux_restore_dirs(rctx, ipsw, aar) + else: + _execute(rctx, "aa patch", [ + "/usr/bin/aa", + "patch", + "-i", + aar, + "-key-value", + "base64:%s" % rctx.attr.archive_decryption_key, + "-src", + "/var/empty", + "-dst", + rctx.path("."), + ]) + restore_dirs = [rctx.path("AssetData/Restore")] + dmg = _single_restore_dmg(restore_dirs) + + extract_args = [ + rctx.path(sevenzip), + "x", + "-y", + dmg, + ] + extract_args.extend(rctx.attr.includes) + _execute(rctx, "7z extract dmg", extract_args) + + # Central cleanup of temporary files and directories after extraction. + # If absent, rctx.delete will handle it gracefully. + rctx.delete("asset.aar") + rctx.delete("AssetData") + rctx.delete("ipsw-output") + rctx.delete("ipsw-config.yaml") + + rctx.template("BUILD.bazel", rctx.attr.build_file) + + return None + +http_aea_archive = repository_rule( + implementation = _http_aea_archive_impl, + attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "archive_decryption_key": attr.string(mandatory = True), + "sha256": attr.string(), + "integrity": attr.string(), + "canonical_id": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "build_file": attr.label( + allow_single_file = True, + ), + "includes": attr.string_list( + default = [], + doc = "Patterns to pass to 7z for selective extraction from the DMG. If empty, extracts everything.", + ), + }, + doc = "Downloads and extracts an AEA-wrapped AppleArchive asset on macOS and Linux AMD64/ARM64.", +) diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index 91b52f5c..968001d6 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -1,46 +1,253 @@ +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") +load( + "@bazel_tools//tools/build_defs/repo:cache.bzl", + "DEFAULT_CANONICAL_ID_ENV", + "get_default_canonical_id", +) load( "@bazel_tools//tools/build_defs/repo:utils.bzl", "get_auth", "patch", + "update_attrs", "workspace_and_buildfile", ) -def _http_deb_archive_impl(rctx): - if rctx.attr.build_file and rctx.attr.build_file_content: - fail("Only one of build_file and build_file_content can be provided.") - download_info = rctx.download_and_extract( - url = rctx.attr.urls, - output = "tmp", - sha256 = rctx.attr.sha256, - type = "deb", - stripPrefix = "", - canonical_id = " ".join(rctx.attr.urls), - auth = get_auth(rctx, rctx.attr.urls), +def _get_source_urls(rctx): + if not rctx.attr.url and not rctx.attr.urls: + fail("At least one of url and urls must be provided") + + source_urls = [] + if rctx.attr.urls: + source_urls = rctx.attr.urls + if rctx.attr.url: + source_urls = [rctx.attr.url] + source_urls + return source_urls + +def _prefixed_pattern(strip_prefix, pattern): + if not strip_prefix: + return pattern + if not pattern: + return strip_prefix + if pattern == strip_prefix or pattern.startswith(strip_prefix + "/"): + return pattern + return strip_prefix + "/" + pattern + +def _downloaded_archive_name(rctx, source_urls): + if rctx.attr.type: + return ".downloaded.archive." + rctx.attr.type + + basename = source_urls[0].split("?", 1)[0].split("/")[-1] + if basename: + return ".downloaded." + basename + return ".downloaded.archive" + +def _placeholder_dir_from_exclude(pattern): + if pattern.endswith("/*"): + return pattern[:-2].strip("/") + return None + +def _ensure_placeholder_dirs(rctx, add_prefix): + for exclude in rctx.attr.excludes: + directory = _placeholder_dir_from_exclude(exclude) + if not directory: + continue + if add_prefix: + directory = add_prefix + "/" + directory + keep_file = directory + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + rctx.delete(keep_file) + +def _host_bsdtar_label(rctx): + platform = repo_utils.platform(rctx) + binary = "tar.exe" if platform.startswith("windows_") else "tar" + return Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary)) + +def _update_integrity_attrs(rctx, attrs, archive_info, remote_files_info, remote_patches_info): + integrity_override = {} + if not rctx.attr.sha256 and not rctx.attr.integrity: + integrity_override["integrity"] = archive_info.integrity + + remote_file_integrity = {path: info.integrity for path, info in remote_files_info.items()} + if rctx.attr.remote_file_integrity != remote_file_integrity: + integrity_override["remote_file_integrity"] = remote_file_integrity + + # TODO(cerisier): Remove this "if" when we no longer support bazel < 8.5.0 + if remote_patches_info: + remote_patch_integrity = {url: info.integrity for url, info in remote_patches_info.items()} + if rctx.attr.remote_patches != remote_patch_integrity: + integrity_override["remote_patches"] = remote_patch_integrity + + if not integrity_override: + return rctx.repo_metadata(reproducible = True) + + return rctx.repo_metadata( + attrs_for_reproducibility = update_attrs(rctx.attr, attrs.keys(), integrity_override), ) +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def symlink_files(ctx): + # type: (repository_ctx) -> None + """Utility function for symlinking local files. + + This is intended to be used in the implementation function of a repository rule. It assumes the + parameter `files` is present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + """ + for path, label in ctx.attr.files.items(): + src_path = ctx.path(label) + + # On Windows `ctx.symlink` may be implemented as a copy, so the file MUST be watched + ctx.watch(src_path) + if not src_path.exists: + fail("Input %s does not exist" % label) + if ctx.path(path).exists: + ctx.delete(path) + ctx.symlink(src_path, path) + +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def download_remote_files(ctx, auth = None): + """Utility function for downloading remote files. + + This rule is intended to be used in the implementation function of + a repository rule. It assumes the parameters `remote_file_urls` and + `remote_file_integrity` to be present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + auth: An optional dict specifying authentication information for some of the URLs. + + Returns: + dict mapping file paths to a download info. + """ + pending = { + path: ctx.download( + remote_file_urls, + path, + canonical_id = ctx.attr.canonical_id, + auth = get_auth(ctx, remote_file_urls) if auth == None else auth, + integrity = ctx.attr.remote_file_integrity.get(path, ""), + block = False, + # Overlaid files may be shell scripts. + executable = True, + ) + for path, remote_file_urls in ctx.attr.remote_file_urls.items() + } + + # Wait until the requests are done + return {path: token.wait() for path, token in pending.items()} + +def _http_deb_archive_impl(rctx): + source_urls = _get_source_urls(rctx) + + if rctx.attr.build_file and rctx.attr.build_file_content: + fail("Only one of build_file and build_file_content can be provided.") + + archive = _downloaded_archive_name(rctx, source_urls) + archive_info = rctx.download( + source_urls, + archive, + rctx.attr.sha256, + canonical_id = rctx.attr.canonical_id or get_default_canonical_id(rctx, source_urls), + auth = get_auth(rctx, source_urls), + integrity = rctx.attr.integrity, + ) + + host_bsdtar = _host_bsdtar_label(rctx) + res = rctx.execute([ + rctx.path(host_bsdtar), + "-xf", + archive, + "--include=data.tar.*", + ]) for ext in ["gz", "xz", "zst"]: - data = "tmp/data.tar.{}".format(ext) + data = "data.tar.{}".format(ext) if rctx.path(data).exists: - rctx.extract( - archive = data, - output = "", - stripPrefix = rctx.attr.strip_prefix, - ) - rctx.delete("tmp") + archive = data break + + strip_prefix = rctx.attr.strip_prefix.strip("/") + includes = [_prefixed_pattern(strip_prefix, include) for include in rctx.attr.includes] + excludes = [_prefixed_pattern(strip_prefix, exclude) for exclude in rctx.attr.excludes] + + if strip_prefix and not includes: + includes = [strip_prefix, strip_prefix + "/*"] + + args = [] + for include in includes: + args.extend(["--include", include]) + for exclude in excludes: + args.extend(["--exclude", exclude]) + if strip_prefix: + args.extend(["--strip-components", str(len(strip_prefix.split("/")))]) + args.extend(rctx.attr.bsdtar_extra_args) + + add_prefix = rctx.attr.add_prefix.strip("/") + keep_file = None + if add_prefix: + keep_file = add_prefix + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + args.extend(["-C", add_prefix]) + + args.extend(["-xf", archive]) + res = rctx.execute([rctx.path(host_bsdtar)] + args) + if res.return_code != 0: + fail("Failed to extract archive: {}\n{}".format(res.stderr, res.stdout)) + + if keep_file: + rctx.delete(keep_file) + _ensure_placeholder_dirs(rctx, add_prefix) + rctx.delete(archive) + workspace_and_buildfile(rctx) - patch(rctx) + remote_files_info = download_remote_files(rctx) + remote_patches_info = patch(rctx) + symlink_files(rctx) + + return _update_integrity_attrs(rctx, _http_deb_archive_attrs, archive_info, remote_files_info, remote_patches_info) + +_http_deb_archive_attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "sha256": attr.string(), + "integrity": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "canonical_id": attr.string(), + "strip_prefix": attr.string(), + "add_prefix": attr.string(default = ""), + "files": attr.string_keyed_label_dict(default = {}), + "type": attr.string(), + "patches": attr.label_list(default = []), + "remote_file_urls": attr.string_list_dict(default = {}), + "remote_file_integrity": attr.string_dict(default = {}), + "remote_module_file_urls": attr.string_list(default = []), + "remote_module_file_integrity": attr.string(default = ""), + "remote_patches": attr.string_dict(default = {}), + "remote_patch_strip": attr.int(default = 0), + "patch_tool": attr.string(default = ""), + "patch_args": attr.string_list(default = []), + "patch_strip": attr.int(default = 0), + "patch_cmds": attr.string_list(default = []), + "patch_cmds_win": attr.string_list(default = []), + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + "workspace_file": attr.label(), + "workspace_file_content": attr.string(), + "includes": attr.string_list(default = []), + "excludes": attr.string_list(default = []), + "bsdtar_extra_args": attr.string_list(default = []), +} http_deb_archive = repository_rule( - _http_deb_archive_impl, - attrs = { - "urls": attr.string_list(mandatory = True), - "sha256": attr.string(mandatory = True), - "strip_prefix": attr.string(), - "build_file": attr.label(allow_single_file = True), - "build_file_content": attr.string(), - "patches": attr.label_list(allow_files = True), - "workspace_file": attr.label(allow_single_file = True), - "workspace_file_content": attr.string(), - }, + implementation = _http_deb_archive_impl, + attrs = _http_deb_archive_attrs, + environ = [DEFAULT_CANONICAL_ID_ENV], ) diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index ac79033c..bafaa6ec 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -1,76 +1,78 @@ def _patchelf_impl(ctx): - output_name = ctx.file.src.basename - if ctx.attr.soname: - output_name = ctx.attr.soname - output = ctx.actions.declare_file("{}/{}".format(ctx.attr.name, output_name)) + output_name = ctx.attr.soname or ctx.file.src.basename + output = ctx.actions.declare_file("{}/{}".format(ctx.label.name, output_name)) - commands = [ - "set -e", - 'cp -f "$2" "$3"', - 'chmod +w "$3"', + inputs = [ctx.file.src] + args = ctx.actions.args() + + # --- soname ----------------------------------------------------------- + if ctx.attr.soname: + args.add("--set-soname", ctx.attr.soname) + + # --- DT_NEEDED -------------------------------------------------------- + args.add_all(ctx.attr.remove_needed, before_each = "--remove-needed") + args.add_all(ctx.attr.add_needed, before_each = "--add-needed") + for old, new in ctx.attr.replace_needed.items(): + args.add("--replace-needed") + args.add(old) + args.add(new) + + # --- RPATH / RUNPATH -------------------------------------------------- + # patchelf only accepts one rpath operation per invocation. + rpath_ops = [ + bool(ctx.attr.set_rpath), + bool(ctx.attr.add_rpath), + ctx.attr.remove_rpath, ] - - if ctx.attr.soname: - commands.append(""" "$1" --set-soname '{}' "$3" """.format(ctx.attr.soname)) - if ctx.attr.remove_needed: - for v in ctx.attr.remove_needed: - commands.append(""" "$1" --remove-needed '{}' "$3" """.format(v)) - if ctx.attr.add_needed: - for v in ctx.attr.add_needed: - commands.append(""" "$1" --add-needed '{}' "$3" """.format(v)) - if ctx.attr.replace_needed: - for k, v in ctx.attr.replace_needed.items(): - commands.append(""" "$1" --replace-needed '{}' '{}' "$3" """.format(k, v)) + if len([op for op in rpath_ops if op]) > 1: + fail("patchelf: set_rpath, add_rpath and remove_rpath are mutually exclusive") if ctx.attr.set_rpath: - commands.append(""" "$1" --set-rpath '{}' "$3" """.format(ctx.attr.set_rpath)) - if ctx.attr.add_rpath: - for path in ctx.attr.add_rpath: - commands.append(""" "$1" --add-rpath '{}' "$3" """.format(path)) - if ctx.attr.remove_rpath: - for path in ctx.attr.remove_rpath: - commands.append(""" "$1" --remove-rpath '{}' "$3" """.format(path)) + args.add("--force-rpath") + args.add("--set-rpath", ctx.attr.set_rpath) + elif ctx.attr.add_rpath: + args.add("--force-rpath") + args.add("--add-rpath", ":".join(ctx.attr.add_rpath)) + elif ctx.attr.remove_rpath: + args.add("--remove-rpath") - renamed_syms = ctx.actions.declare_file("{}.rename.txt".format(ctx.label.name)) + # --- dynamic symbol renames ------------------------------------------ if ctx.attr.rename_dynamic_symbols: - content = "\n".join([ - "{} {}".format(k, v) - for k, v in ctx.attr.rename_dynamic_symbols.items() - ]) - ctx.actions.write(renamed_syms, content) - commands.append(""" "$1" --rename-dynamic-symbols '{}' "$3" """.format(renamed_syms.path)) - else: - ctx.actions.write(renamed_syms, "") + symbols = ctx.actions.declare_file(ctx.label.name + ".symbols") + ctx.actions.write(symbols, "".join([ + "{} {}\n".format(old, new) + for old, new in ctx.attr.rename_dynamic_symbols.items() + ])) + inputs.append(symbols) + args.add("--rename-dynamic-symbols", symbols) - ctx.actions.run_shell( - inputs = [ctx.file.src, renamed_syms], + args.add("--output", output) + args.add(ctx.file.src) + + ctx.actions.run( + executable = ctx.executable._patchelf, + inputs = inputs, outputs = [output], - arguments = [ctx.executable._patchelf.path, ctx.file.src.path, output.path], - command = "\n".join(commands), - tools = [ctx.executable._patchelf], + arguments = [args], + mnemonic = "Patchelf", + progress_message = "Patching ELF %{output}", ) - - return [ - DefaultInfo( - files = depset([output]), - ), - ] + return [DefaultInfo(files = depset([output]))] patchelf = rule( implementation = _patchelf_impl, attrs = { "src": attr.label(allow_single_file = True, mandatory = True), - "soname": attr.string(), + "soname": attr.string(doc = "Set DT_SONAME; also used as the output file name."), "add_needed": attr.string_list(), "remove_needed": attr.string_list(), - "replace_needed": attr.string_dict(), - "rename_dynamic_symbols": attr.string_dict(), + "replace_needed": attr.string_dict(doc = "old -> new DT_NEEDED replacements."), + "rename_dynamic_symbols": attr.string_dict(doc = "old -> new symbol renames."), "set_rpath": attr.string(), - "add_rpath": attr.string_list(), - "remove_rpath": attr.string_list(), + "add_rpath": attr.string_list(doc = "Entries appended to the existing RPATH (joined with ':')."), + "remove_rpath": attr.bool(default = False, doc = "Strip the DT_RPATH/DT_RUNPATH entry entirely."), "_patchelf": attr.label( default = "@patchelf", - allow_single_file = True, executable = True, cfg = "exec", ), diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c new file mode 100644 index 00000000..770a4b91 --- /dev/null +++ b/bazel/runfiles_init.c @@ -0,0 +1,15 @@ +extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); + +// Preserve the existing startup ABI. Registration happens in C; +// runfiles discovery and initialization remain in Zig. +// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. +// The original Zig initializer already made this assumption to read argv[0]. +// +// Let C emit the platform's constructor registration. Zig's incremental ELF +// linker does not yet handle .init_array's section type, flags, and size correctly. +// TODO: Revert this workaround once Zig issue #37453 is fixed: +// https://codeberg.org/ziglang/zig/issues/37453 +__attribute__((constructor)) +void _bazel_runfiles_init(int argc, char **argv, char **envp) { + _bazel_runfiles_init_impl(argc, argv, envp); +} diff --git a/bazel/stamp/BUILD.bazel b/bazel/stamp/BUILD.bazel new file mode 100644 index 00000000..387f189c --- /dev/null +++ b/bazel/stamp/BUILD.bazel @@ -0,0 +1,37 @@ +load("@bazel_lib//lib:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +run_binary( + name = "stamp_cc_src", + outs = ["stamp.c"], + args = [ + "$(location stamp.c)", + "bazel-out/stable-status.txt", + "bazel-out/volatile-status.txt", + ], + stamp = 1, + tool = ":genstamp", + visibility = ["//visibility:private"], +) + +cc_binary( + name = "genstamp", + srcs = ["genstamp.c"], + visibility = ["//visibility:private"], +) + +cc_library( + name = "cc", + srcs = [":stamp_cc_src"], + visibility = ["//visibility:public"], +) + +zig_library( + name = "zig", + import_name = "bazel/stamp", + main = "stamp.zig", + visibility = ["//visibility:public"], + deps = [":cc"], +) diff --git a/bazel/stamp/genstamp.c b/bazel/stamp/genstamp.c new file mode 100644 index 00000000..4fba8e6f --- /dev/null +++ b/bazel/stamp/genstamp.c @@ -0,0 +1,121 @@ +#include +#include +#include + +static void write_escaped(FILE *out, const char *value) { + for (const unsigned char *p = (const unsigned char *)value; *p != '\0'; ++p) { + switch (*p) { + case '\\': + fputs("\\\\", out); + break; + case '"': + fputs("\\\"", out); + break; + case '\n': + fputs("\\n", out); + break; + case '\r': + fputs("\\r", out); + break; + case '\t': + fputs("\\t", out); + break; + default: + fputc(*p, out); + break; + } + } +} + +static int process_stream(FILE *out, FILE *in, const char *path) { + char *line = NULL; + size_t cap = 0; + ssize_t len; + + while ((len = getline(&line, &cap, in)) != -1) { + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { + line[--len] = '\0'; + } + + if (len == 0) { + continue; + } + + char *space = strchr(line, ' '); + if (space == NULL) { + continue; + } + + *space = '\0'; + const char *name = line; + const char *value = space + 1; + + if (*name == '\0') { + continue; + } + + fputs("const char ", out); + fputs(name, out); + fputs("[] = \"", out); + write_escaped(out, value); + fputs("\";\n", out); + } + + if (ferror(in)) { + fprintf(stderr, "genstamp: failed while reading %s\n", path); + free(line); + return 1; + } + + free(line); + return 0; +} + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s [ ...]\n", argv[0]); + return 1; + } + + const char *output_path = argv[1]; + FILE *out = fopen(output_path, "w"); + if (out == NULL) { + fprintf(stderr, "genstamp: failed to open %s for writing\n", output_path); + return 1; + } + + for (int i = 2; i < argc; ++i) { + const char *path = argv[i]; + FILE *in = fopen(path, "r"); + if (in == NULL) { + fprintf(stderr, "genstamp: failed to open %s\n", path); + fclose(out); + return 1; + } + + int rc = process_stream(out, in, path); + if (fclose(in) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", path); + fclose(out); + return 1; + } + + if (rc != 0) { + fclose(out); + return rc; + } + } + + if (fflush(out) != 0) { + fprintf(stderr, "genstamp: failed to flush %s\n", output_path); + fclose(out); + return 1; + } + + if (fclose(out) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", output_path); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/bazel/stamp/stamp.zig b/bazel/stamp/stamp.zig new file mode 100644 index 00000000..0e60e2c7 --- /dev/null +++ b/bazel/stamp/stamp.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn stamp(comptime name: []const u8) []const u8 { + return std.mem.span(@extern([*c]const u8, .{ .name = name })); +} diff --git a/bazel/swig.bzl b/bazel/swig.bzl index d81310fb..cd4f994f 100644 --- a/bazel/swig.bzl +++ b/bazel/swig.bzl @@ -1,6 +1,7 @@ load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_common") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain", "use_cc_toolchain") def _swig_cc_library_impl(ctx): @@ -60,6 +61,7 @@ def _swig_cc_library_impl(ctx): ] args.add("-o", output_cpp) args.add("-w-305") + args.add("-w-524") args.add(ctx.file.interface) inputs = depset(ctx.attr.srcs, transitive = [ diff --git a/bazel/zig_index.json b/bazel/zig_index.json index da8c0455..b8cf9b9e 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,133 +1,518 @@ { + "0.17.0": { + "version": "0.17.0", + "date": "2026-10-01", + "docs": "https://ziglang.org/documentation/0.17.0/", + "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", + "notes": "https://ziglang.org/download/0.17.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", + "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", + "size": "23049960" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", + "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", + "size": "57030388" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "size": "59317572" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "size": "53985220" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "size": "57332648" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "size": "52877280" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", + "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", + "size": "53841896" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", + "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", + "size": "57116976" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", + "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", + "size": "56990076" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", + "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", + "size": "59823964" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", + "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", + "size": "54257324" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", + "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", + "size": "57136832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "size": "100268137" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "size": "95992853" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", + "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", + "size": "101891323" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", + "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", + "size": "52860244" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", + "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", + "size": "54273908" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", + "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", + "size": "57027652" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", + "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", + "size": "57199160" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", + "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", + "size": "57468884" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", + "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", + "size": "52769844" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", + "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", + "size": "55345456" + }, + "riscv64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", + "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", + "size": "57157568" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", + "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", + "size": "60380332" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", + "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", + "size": "57363300" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", + "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", + "size": "53265892" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", + "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", + "size": "54024240" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", + "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", + "size": "57490512" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", + "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", + "size": "58612444" + } + }, + "0.17.0-dev.1936+5a625d5f3": { + "version": "0.17.0-dev.1936+5a625d5f3", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", + "size": "59162280" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", + "size": "53867184" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", + "size": "57180268" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", + "size": "52741876" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", + "size": "100793891" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", + "size": "96526396" + } + }, + "0.17.0-dev.1509+bb296ab9b": { + "version": "0.17.0-dev.1509+bb296ab9b", + "date": "2026-07-29", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", + "size": "59064872" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", + "size": "53773824" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", + "size": "57056988" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", + "size": "52649416" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", + "size": "100743514" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", + "size": "96467368" + } + }, "master": { - "version": "0.16.0-dev.2611+f996d2866", - "date": "2026-02-14", + "version": "0.16.0-dev.3153+d6f43caad", + "date": "2026-04-12", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "cf085b0b63b3520cda55aa7d98a8c86b28a5c9739bffe4a0253c58880ded4c7b", - "size": "22442124" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "cb598f754d00a71af7586d437f490056b084eb6e54587fc2bb9f55148757fd49", + "size": "22477252" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "b87fcb4aa966c9b7f07ceca302b15eb587936938a65b5861188200d5b3616c0d", - "size": "55205844" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "a2ccc95aaed569b3c25d8693f01187144f09e08a2a9b828bad3e5f2c19a4c2bc", + "size": "55240588" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", - "size": "57619112" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", + "size": "57381556" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", - "size": "52470216" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", + "size": "52217412" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", - "size": "55505692" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", + "size": "55445192" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", - "size": "51276904" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", + "size": "51193908" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "b0b809695bee67798ce9c956be24d850beab5b2ca4f3bf3fc65552baea0214bb", - "size": "52142540" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "aec322b170a88c998c1c7b59c731e808543310008960b1654caab569567e7c95", + "size": "52093456" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "9244e44969386bd35477079b42db7c7dd5b636ed6ddd21cbcb35debf1db5f285", - "size": "55318440" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "ad83132048c075edd1031a1e2fc9b29921e342f6b1d58142923e60e203a5b53a", + "size": "55303468" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "fd9f8b35403ac5c78d40c979a0f467413ddbb0294c2157590bb8e48635c18df5", - "size": "55350924" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1866a073fdcd0773397d345b21f98221a20126f7e1cec298c16ff0f4ca3fb78f", + "size": "55241328" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "24d2fbc05b7d92a63e228292c693f7fdfa974655510e8c7089e09ce8b9f41fd4", - "size": "58210984" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "6572c8768de14c0a0bbaf1a0401e19849681840594744a875ade93e4dc040edd", + "size": "58171580" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "1448d4e780a1257d042c836c237807c6ecaaba36c03961fef1940a842c22cd6d", - "size": "52566124" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9bdf0f91187ede6b418b898d178521c4ce2065171b8bcfaa68cae0adf1d0a8f0", + "size": "52511048" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "d5bd0131aa1e1a40434acc703afd48f0d235ba9d1e988c944148d03df4dce967", - "size": "54992952" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c893299d0e486337390262ef480615d4f573da158f73a647eb104e10a2811f0d", + "size": "54926720" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", - "size": "98104739" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", + "size": "97965399" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", - "size": "94105952" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", + "size": "93857928" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2611+f996d2866.zip", - "shasum": "ccc288a50431d732b4b8dcc55c6e76ec79b406cbb73c794864cd3c538a77ee0a", - "size": "99920751" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "51d579299617bd5fae4d0333d6f2b06a8078c1a6f2f6db2dd08289236fad70d4", + "size": "99767726" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "c0c4b7bee7d67c4fb44692a96068bde05ba88e4bbe431928540db5bb29a8ab34", - "size": "51220656" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "fe2a97ed29641cc214d58fa2c054fb3ad3a7120b7b43241e6b833c0978187bb6", + "size": "51125424" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "9f470b7304d70e59febfb23e14cacdbf1f81c2839e181f290009c3ccb04bbd4e", - "size": "52814796" - }, - "powerpc64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "7caa39c909cee4342876f0623a40ad91f364b509a14bdcdcdd86629071a1dfbb", - "size": "53925668" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0aa627f73f0bc9c100394d512a375188afb8d33168409050d07b0c8b87adbc0b", + "size": "52702132" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "4a0ea164b83b6bac83764c5b34c64b85ecbae1b685dc07a0fa59410efdde32c5", - "size": "55367852" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "18500e3b2e742c07541b2d35377e3dc770758ea41e9a3bf09535eacad5266583", + "size": "55254720" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "49f2e4403aad2256b921a2952a7e4e262c1077f3eb3bb0bd7c3e69a3154fbf68", - "size": "55478468" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "593c19b8feb76a2e66c6d49f3a96390eed22fc4e2c780a96429d202bf4912afc", + "size": "55393696" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "169314c88df8e98c4b5c2db9805d160dcf332013fcd1f8a0df09f81a5eb3425a", - "size": "55617364" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "b043fa6731e205f208d2e50fa84d1d79c7060e04688a4f5e3bff6dace337d5f8", + "size": "55587512" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "a091fd4f7128e0098915d6693eb720b1a4d1740dcb552e99a0957d99c3fda3c4", - "size": "51174248" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c1625244d6fe4008feb27d8bc7bf49739da98d04bc961295124a4e8732cf3b18", + "size": "51092376" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "e399f1751fc25e3e04c2b7411a5fd9b9c316233b9370fc2fdf7fb45e8b020922", - "size": "53840612" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "582116aeee90ea3d38ede6901f93138e543a07c6dc32d2645bf0b4e6fee813c6", + "size": "53732592" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "8fae56f0683a8f9d2767ad89adc156971f0a872fb485b4c70966e6b113bcad68", - "size": "58800484" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "eba2eb8e8efe0748820fcedb4066233a96406f97c3fa7b326e220f7c3a9e07f3", + "size": "58765376" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", - "shasum": "0063013d3d306b4979b16f87db42fa1c7d94d5eac9275979cf40afa944328a56", - "size": "55551904" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "281d172d159326167a75f2bc87fbe18afe4e37a26b5a1440cd65c8eef952626a", + "size": "55515836" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1d750bb3ed57ff4a83b1411a07df7e38f86a0a611123fcbede1ed84a48937585", + "size": "51551144" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "2bd936eff1c2fadd5aa842e30f35f8509ee07f980bcb42a407940b1ae8fe5f29", + "size": "52270724" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c2067369e07e991f2e1cfe8f657c9185d726487a4ab4cbc1fde9768c8fae363d", + "size": "55713660" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "01122836baaa88627de2e82addf7cdcdcfb3ba104ae9d051d92e36aaad4f4e27", + "size": "56927908" + } + }, + "0.16.0": { + "version": "0.16.0", + "date": "2026-04-13", + "docs": "https://ziglang.org/documentation/0.16.0/", + "stdDocs": "https://ziglang.org/documentation/0.16.0/std/", + "notes": "https://ziglang.org/download/0.16.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.16.0/zig-0.16.0.tar.xz", + "shasum": "43186959edc87d5c7a1be7b7d2a25efffd22ce5807c7af99067f86f99641bfdf", + "size": "22503260" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.16.0/zig-bootstrap-0.16.0.tar.xz", + "shasum": "2a8266a4205772ef40838c8cbdf14875855a515ff3adf89b49c2d2ae93613d10", + "size": "55245980" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", + "shasum": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + "size": "57396836" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", + "shasum": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + "size": "52238004" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", + "shasum": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + "size": "55478392" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", + "shasum": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + "size": "51211944" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-linux-0.16.0.tar.xz", + "shasum": "f85116bf2f9189bb6ae280c7f92f03b89c2551a88e17881c0c2df86bf4e42c50", + "size": "52122000" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-linux-0.16.0.tar.xz", + "shasum": "bc069b0f2f568f54bafbdfc1d65b12fd386ed6a652044a37aee6a4f72f14076e", + "size": "55313616" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-linux-0.16.0.tar.xz", + "shasum": "18800b45c08bf40b335ca5ab79aea70aca287ca969036e938155772becaeebeb", + "size": "55242728" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-linux-0.16.0.tar.xz", + "shasum": "4e34e279a9f856358de420490b531974c3d37f8f3707eef9f0342e92c14c301f", + "size": "58196012" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-loongarch64-linux-0.16.0.tar.xz", + "shasum": "2503be8ecc5965f1f7962471d267d9f83fcb3cc2f7ff78ac34093b9722bbea93", + "size": "52515080" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-s390x-linux-0.16.0.tar.xz", + "shasum": "14db4884b31414bd637879036c1e6e335f3ba44ccac59df104163728a0cab30e", + "size": "54995144" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", + "shasum": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", + "size": "97217739" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", + "shasum": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", + "size": "93109828" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-windows-0.16.0.zip", + "shasum": "8aee7e8a8deb998ba96cb95d89aa5fcdf32933fdc67de51d280d9e4d7396f1a0", + "size": "99015159" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-freebsd-0.16.0.tar.xz", + "shasum": "0a441f50696a34cfb9f9a0b3c510fd49a56b18b364e03c26525cb7999959c969", + "size": "51137568" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-freebsd-0.16.0.tar.xz", + "shasum": "78b2a7bfa74f504075dd4b5c00418347a21b893f0eae580cf4e56c553cf181f6", + "size": "52741336" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-freebsd-0.16.0.tar.xz", + "shasum": "46b3fa0239be42669d86ce8394f8de2d608a4f51e3e65cb8222bfd543b95e310", + "size": "55232224" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-freebsd-0.16.0.tar.xz", + "shasum": "aefbaea44f32bde07118f5522b7bb21de9967a6d09183e3e37b1f5be2c78f948", + "size": "55416052" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-freebsd-0.16.0.tar.xz", + "shasum": "390379bfaf6b89b001c6bb4035b8fcbc5c12193fd9fd5048c3a0fe39e5d1cf72", + "size": "55591888" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-netbsd-0.16.0.tar.xz", + "shasum": "4767425d72c779275ebbc5a6319f5a3c5d109664d3d8c880125d550e87d9d46a", + "size": "51104512" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-netbsd-0.16.0.tar.xz", + "shasum": "07aaa403c18e9fa33687a3eb3109bb484a96aece95a4f6792a911d2eefac07c5", + "size": "53772868" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-netbsd-0.16.0.tar.xz", + "shasum": "aa36c02caad52d92e6354baec16dd866642e64b7c30e4d515806c8296d65586f", + "size": "58792260" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-netbsd-0.16.0.tar.xz", + "shasum": "e235ca96f63034a009aca831c69394a1c14498a8e06ff5f358fc3fb4bed73dfb", + "size": "55533332" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-openbsd-0.16.0.tar.xz", + "shasum": "f3c868ccc1bc30e2d2b68128186b2153d86c3efc41190f4170a43024a3cd7809", + "size": "51564888" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-openbsd-0.16.0.tar.xz", + "shasum": "33786d6487aaeca9f300e5a932d5ee5365d9254be0add44b3af8f6db8441ca2f", + "size": "52301840" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-openbsd-0.16.0.tar.xz", + "shasum": "ffb74aefa8184a4b886f8c0a9105db00ac9562d502330f4e764a7a49171e4f8e", + "size": "55734756" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-openbsd-0.16.0.tar.xz", + "shasum": "25b72946e8b2dde5c93bc03cbebeb17e4d90eea8040acce8bb78cefdeee97337", + "size": "56962688" } }, "0.15.2": { diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl new file mode 100644 index 00000000..85938a0c --- /dev/null +++ b/bazel/zig_options.bzl @@ -0,0 +1,170 @@ +"""Build settings for Zig options.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_zig//zig:defs.bzl", "zig_library") + +ZigOption = provider( + doc = "A Zig option's configured value and allowed string values.", + fields = { + "name": "The flag's target name, or this option's target name for explicit values.", + "module_name": "The Bazel module declaring the flag, or empty when unavailable.", + "value": "The configured build setting or explicit value.", + "type": "The Zig integer type, or None for other flag types.", + "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", + "enum_name": "Exported enum type name for a constrained string flag; empty keeps the option as a string.", + "nullable": "Whether a string or enum emits null for an empty value.", + }, +) + +_FlagInfo = provider(fields = ["allowed_values"]) + +def _flag_info_impl(_target, ctx): + return [_FlagInfo(allowed_values = getattr(ctx.rule.attr, "values", []))] + +_flag_info = aspect(implementation = _flag_info_impl) + +def _zig_option_impl(ctx): + flag = ctx.attr.flag + if ctx.attr.value_source == "flag": + if flag == None: + fail("flag must resolve to a build setting") + value = flag[BuildSettingInfo].value + else: + value = getattr(ctx.attr, ctx.attr.value_source) + if not (types.is_bool(value) or types.is_int(value) or types.is_string(value)): + fail("Zig options require an integer, boolean, or string value") + allowed_values = None + if types.is_string(value): + allowed_values = flag[_FlagInfo].allowed_values if flag else [] + if ctx.attr.type and not types.is_int(value): + fail("type is only supported for integer options") + if ctx.attr.nullable and not types.is_string(value): + fail("nullable is only supported for string options") + if ctx.attr.enum_name and not allowed_values: + fail("enum_name requires a string flag with nonempty values") + return [ZigOption( + name = flag.label.name if flag else ctx.label.name, + module_name = "", + value = value, + type = (ctx.attr.type or "usize") if types.is_int(value) else None, + allowed_values = allowed_values, + enum_name = ctx.attr.enum_name, + nullable = ctx.attr.nullable, + )] + +def _zig_string(value): + # JSON and Zig differ in their control-character and Unicode escapes. + escaped = [] + for char in value.elems(): + part = json.encode(char)[1:-1] + if part.startswith("\\u"): + part = "\\u{" + part[2:] + "}" + escaped.append(part.replace("\\b", "\\x08").replace("\\f", "\\x0c")) + return '"' + "".join(escaped) + '"' + +def _zig_options_impl(ctx): + lines = [] + names = {} + for setting in ctx.attr.settings: + option = setting[ZigOption] + name = option.name + if name in names: + fail("Duplicate Zig declaration: {}".format(name)) + names[name] = True + value = option.value + if types.is_bool(value): + zig_type = "bool" + literal = "true" if value else "false" + elif types.is_int(value): + zig_type = option.type + literal = str(value) + elif option.enum_name: + enum_values = [v for v in option.allowed_values if not option.nullable or v != ""] + zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in enum_values]) + " }" + if option.enum_name in names: + fail("Duplicate Zig declaration: {}".format(option.enum_name)) + names[option.enum_name] = True + lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) + zig_type = "@" + _zig_string(option.enum_name) + if option.nullable: + zig_type = "?" + zig_type + literal = "null" if option.nullable and not value else ".@" + _zig_string(value) + else: + zig_type = "?[]const u8" if option.nullable else "[]const u8" + literal = "null" if option.nullable and not value else _zig_string(value) + lines.append("pub const @{}: {} = {};".format(_zig_string(name), zig_type, literal)) + ctx.actions.write(ctx.outputs.out, "\n".join(lines) + "\n") + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +_zig_options_source = rule( + implementation = _zig_options_impl, + attrs = { + "out": attr.output(mandatory = True), + "settings": attr.label_list( + mandatory = True, + allow_files = False, + providers = [ZigOption], + ), + }, +) + +def zig_options(name, settings, **kwargs): + """Generate an options module and expose it as a zig_library. + + Args: + name: Library target name; the generated source is .zig. + settings: zig_option targets; flag names or explicit option names become quoted Zig constants. + **kwargs: Attributes forwarded to zig_library, except main which is generated. + """ + if "main" in kwargs: + fail("zig_options generates its own main source") + source_attrs = {key: kwargs[key] for key in ["testonly", "tags", "compatible_with", "restricted_to", "target_compatible_with"] if key in kwargs} + _zig_options_source( + name = name + "_source", + settings = settings, + out = name + ".zig", + visibility = ["//visibility:private"], + **source_attrs + ) + zig_library(name = name, main = ":" + name + "_source", **kwargs) + +_zig_option = rule( + implementation = _zig_option_impl, + attrs = { + "flag": attr.label( + providers = [BuildSettingInfo], + aspects = [_flag_info], + doc = "Boolean, integer, or string build setting. An aspect discovers allowed string values.", + ), + "bool": attr.bool(), + "string": attr.string(), + "int": attr.int(), + "value_source": attr.string(mandatory = True, values = ["flag", "bool", "string", "int"]), + "type": attr.string(doc = "Zig integer type; defaults to usize. Only valid for integer options."), + "nullable": attr.bool(doc = "Emit an optional string or enum; an empty value becomes null. String flags with values must include an empty string to allow null; it is excluded from the enum."), + "enum_name": attr.string(doc = "Emit a named Zig enum instead of a string; requires nonempty values. Defaults to emitting a string."), + }, + doc = "Expose a build setting or explicit value as a Zig option, named after the flag or this target respectively.", +) + +def zig_option(name, flag = None, bool = None, string = None, int = None, **kwargs): + """Expose exactly one build setting or explicit value as a Zig option. + + Args: + name: Target name; also the generated Zig constant name for explicit values. + flag: Boolean, integer, or string build setting to read. + bool: Explicit boolean value, which may use select(). + string: Explicit string value, which may use select(). + int: Explicit integer value, which may use select(). + **kwargs: Attributes forwarded to the rule, including type, nullable, and enum_name. + """ + sources = {key: value for key, value in { + "flag": flag, + "bool": bool, + "string": string, + "int": int, + }.items() if value != None} + if len(sources) != 1: + fail("zig_option requires exactly one of flag, bool, string, or int") + _zig_option(name = name, value_source = sources.keys()[0], **dict(sources, **kwargs)) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index ee4adb36..96bebfab 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,4 +1,5 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") load("@rules_zig//zig:defs.bzl", "zig_static_library") def zig_srcs(name, zig_bin = "", zig_lib = ""): diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel new file mode 100644 index 00000000..077f5b34 --- /dev/null +++ b/bin/zml-smi/BUILD.bazel @@ -0,0 +1,222 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//tools/release:hashes.bzl", "hashes") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") + +# zig libraries + +# smi + +run_binary( + name = "libnrt.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], + outs = ["libnrt.so.1"], + args = [ + "--strip-debug", + "-o", + "$(location libnrt.so.1)", + "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", + ], + target_compatible_with = ["@platforms//os:linux"], + tool = "@llvm//tools:llvm-strip", +) + +run_binary( + name = "libncfw.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libncfw.patchelf"], + outs = ["libncfw.so.2"], + args = [ + "--strip-debug", + "-o", + "$(location libncfw.so.2)", + "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", + ], + target_compatible_with = ["@platforms//os:linux"], + tool = "@llvm//tools:llvm-strip", +) + +copy_to_directory( + name = "sandbox", + srcs = select({ + "@platforms//os:macos": [], + "@llvm//platforms/config:linux_x86_64": [ + ":libncfw.stripped", + ":libnrt.stripped", + "//platforms/rocm:zmlxrocm", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-sysdeps//:system_libs", + "@libzml_rocm//:amdgpu_ids", + ], + "@llvm//platforms/config:linux_aarch64": [], + }), + include_external_repositories = ["**"], + replace_prefixes = { + "amdgpu.ids": "share/libdrm/amdgpu.ids", + "platforms/rocm": "lib", + "libnrt.so.1": "lib/libnrt.so.1", + "libncfw.so.2": "lib/libncfw.so.2", + }, +) + +run_binary( + name = "zml-smi.stripped", + srcs = [":zml-smi"], + outs = ["zml-smi.stripped"], + args = [ + "--strip-all", + "-o", + "$(location zml-smi.stripped)", + "$(location :zml-smi)", + ], + tool = "@llvm//tools:llvm-strip", +) + +zig_binary( + name = "zml-smi", + data = [":sandbox"], + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/api", + "//bin/zml-smi/collector", + "//bin/zml-smi/csv", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", + "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", + "//stdx", + "@libvaxis//:vaxis", + "@rules_zig//zig/runfiles", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], + }) + select({ + "@llvm//platforms/config:linux_x86_64": [ + "//bin/zml-smi/platforms/amdsmi", + "//bin/zml-smi/platforms/neuron", + "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/oneapi", + "//bin/zml-smi/platforms/tpu", + "//platforms/neuron:libnrt_headers", + ], + "@llvm//platforms/config:linux_aarch64": [ + "//bin/zml-smi/platforms/nvml", + ], + "//conditions:default": [], + }), +) + +# docker image + +mtree_spec( + name = "mtree_spec", + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], + tags = ["manual"], +) + +# Use this genrule until mtree_mutate is fixed +genrule( + name = "mtree", + srcs = [":mtree_spec"], + outs = ["mtree.txt"], + cmd = """sed -e '/type=dir/d' -e '/zml-smi\\/zml-smi /d' -e 's#^bin/##g' -e 's#zml-smi\\.stripped#zml-smi#' -e '/stamp.c/d' $< > $@""", + tags = ["manual"], +) + +# mtree_mutate( +# name = "mtree", +# mtree = ":mtree_spec", +# strip_prefix = "bin", +# ) + +# Release archives, one per platform, correctly named for distribution +# Build one: bazel build --config=release //bin/zml-smi:zml-smi-linux-amd64 +# Build all: bazel build --config=release //bin/zml-smi:archives + +_RELEASE_PLATFORMS = [ + ("linux", "amd64"), + ("linux", "arm64"), + ("macos", "arm64"), +] + +[tar( + name = "archive_" + os + "_" + arch, + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], + out = "zml-smi-" + os + "-" + arch + ".tar.zst", + args = [ + "--options", + ",".join([ + "zstd:compression-level=22", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +[platform_transition_filegroup( + name = "zml-smi-" + os + "-" + arch, + srcs = [":archive_" + os + "_" + arch], + tags = ["manual"], + target_platform = "//platforms:" + os + "_" + arch, +) for os, arch in _RELEASE_PLATFORMS] + +[hashes( + name = "zml-smi-" + os + "-" + arch + ".sha256", + src = ":zml-smi-" + os + "-" + arch, + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +sh_binary( + name = "release", + srcs = ["scripts/release.sh"], + tags = ["manual"], +) + +filegroup( + name = "archives", + srcs = [":zml-smi-" + os + "-" + arch for os, arch in _RELEASE_PLATFORMS] + + [":zml-smi-" + os + "-" + arch + ".sha256" for os, arch in _RELEASE_PLATFORMS], + tags = ["manual"], +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["/zml-smi/zml-smi"], + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [ + ":archive_linux_amd64", + ], +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/zml-smi:latest"], + tags = ["manual"], +) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel new file mode 100644 index 00000000..1a09f597 --- /dev/null +++ b/bin/zml-smi/api/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "api", + import_name = "zml-smi/api", + main = "api.zig", + srcs = [ + "client.zig", + "server.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/utils:double_buffer", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig new file mode 100644 index 00000000..fb39ddd8 --- /dev/null +++ b/bin/zml-smi/api/api.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig new file mode 100644 index 00000000..ddec6d2d --- /dev/null +++ b/bin/zml-smi/api/client.zig @@ -0,0 +1,138 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +fn setRemote(device: *DeviceInfo) void { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (device.* == @field(smi_info.device_info.Target, field_name)) { + @field(device, field_name).values[0].remote = true; + @field(device, field_name).values[1].remote = true; + } + } +} + +pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { + var it = std.mem.splitScalar(u8, hosts, ','); + while (it.next()) |host| { + addHost(collector, host) catch |err| { + std.log.err("{s}: {s}", .{ host, @errorName(err) }); + }; + } +} + +fn addHost(collector: *Collector, host: []const u8) !void { + const body = try httpGet(collector.gpa, collector.io, host); + defer collector.gpa.free(body); + + const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var devices: std.ArrayList(*DeviceInfo) = .empty; + for (json_devices) |item| { + var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + const info = try collector.addDevice(device); + try devices.append(collector.arena, info); + } + + if (devices.items.len == 0) { + return; + } + + const processes = try collector.createProcessList(); + const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + const back = processes.back(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(collector.gpa, proc) catch continue; + } + processes.swap(); + + const url = try collector.arena.dupe(u8, host); + const poll_arena = try collector.createPollArena(); + try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); +} + +fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { + _ = poll_arena.reset(.retain_capacity); + + const body = httpGet(gpa, io, host) catch return; + defer gpa.free(body); + + const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + for (json_devices, 0..) |item, i| { + if (i >= devices.len) { + std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); + break; + } + var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { + @field(devices[i], field_name).back().* = @field(device, field_name).front().*; + @field(devices[i], field_name).swap(); + } + } + } + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + _ = str_arenas[back_idx].reset(.retain_capacity); + const alloc = str_arenas[back_idx].allocator(); + const back = processes.back(); + back.clearRetainingCapacity(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(gpa, proc) catch continue; + } + processes.swap(); +} + +fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { + if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { + std.log.err("remote host must start with http:// or https://: {s}", .{host}); + return error.InvalidUrl; + } + + const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); + defer allocator.free(url); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var aw: std.Io.Writer.Allocating = .init(allocator); + errdefer aw.deinit(); + + const result = try client.fetch(.{ + .location = .{ .url = url }, + .response_writer = &aw.writer, + }); + + if (result.status != .ok) { + aw.deinit(); + return error.HttpError; + } + + return try aw.toOwnedSlice(); +} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig new file mode 100644 index 00000000..e05aa7fe --- /dev/null +++ b/bin/zml-smi/api/server.zig @@ -0,0 +1,105 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); +const json = @import("zml-smi/json"); +const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const Server = struct { + allocator: std.mem.Allocator, + io: std.Io, + devices: []const *DeviceInfo, + process_lists: []const *const ProcessDoubleBuffer, + enricher: *ProcessEnricher, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { + const address: std.Io.net.IpAddress = .{ .ip4 = .{ + .bytes = .{ 0, 0, 0, 0 }, + .port = port, + } }; + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .allocator = allocator, + .io = io, + .devices = devices, + .process_lists = process_lists, + .enricher = enricher, + .tcp = tcp, + }; + } + + pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { + var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { + std.log.err("api server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + var procs: std.ArrayList(ProcessInfo) = .empty; + defer procs.deinit(self.allocator); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + procs.clearRetainingCapacity(); + for (self.process_lists) |pl| { + procs.appendSlice(self.allocator, pl.front().items) catch {}; + } + self.enricher.enrich(self.io, procs.items); + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, + }, + }, + }); + + try json.write(&body_writer.writer, self.devices, procs.items); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel new file mode 100644 index 00000000..0bf5fa8e --- /dev/null +++ b/bin/zml-smi/collector/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "collector", + import_name = "zml-smi/collector", + main = "collector.zig", + srcs = ["worker.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig new file mode 100644 index 00000000..d98b204d --- /dev/null +++ b/bin/zml-smi/collector/collector.zig @@ -0,0 +1,93 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("worker.zig").Worker; + +pub const Collector = struct { + device_infos: std.ArrayList(*DeviceInfo) = .empty, + process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, + poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, + arena: std.mem.Allocator, + gpa: std.mem.Allocator, + worker: Worker, + io: std.Io, + poll_only: bool, + + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { + return .{ + .arena = arena, + .gpa = gpa, + .io = io, + .worker = .{ .poll_interval_ms = options.poll_interval_ms }, + .poll_only = options.poll_only, + }; + } + + pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { + const info = try self.arena.create(DeviceInfo); + info.* = initial; + try self.device_infos.append(self.arena, info); + + return info; + } + + pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { + const list = try self.arena.create(ProcessDoubleBuffer); + list.* = .{ .values = .{ .empty, .empty } }; + try self.process_lists.append(self.arena, list); + + return list; + } + + pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { + const poll_arena = try self.arena.create(std.heap.ArenaAllocator); + poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); + try self.poll_arenas.append(self.arena, poll_arena); + + return poll_arena; + } + + pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { + @call(.auto, pollOnce, args); + if (self.poll_only) return; + + const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); + try self.worker.spawn(self.io, struct { + fn f(io: std.Io, w: *const Worker, a: Args) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + + @call(.auto, pollOnce, a); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } + } + }.f, .{ self.io, &self.worker, args }); + } + + pub fn deinit(self: *Collector) void { + self.worker.shutdown(self.io); + + for (self.device_infos.items) |info| { + self.arena.destroy(info); + } + self.device_infos.deinit(self.arena); + + for (self.process_lists.items) |list| { + list.values[0].deinit(self.gpa); + list.values[1].deinit(self.gpa); + } + self.process_lists.deinit(self.arena); + + for (self.poll_arenas.items) |poll_arena| { + poll_arena.deinit(); + } + self.poll_arenas.deinit(self.arena); + } +}; diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig new file mode 100644 index 00000000..125440c5 --- /dev/null +++ b/bin/zml-smi/collector/worker.zig @@ -0,0 +1,25 @@ +const std = @import("std"); + +pub const Worker = struct { + poll_interval_ms: u16, + should_stop: std.atomic.Value(bool) = .init(false), + group: std.Io.Group = .init, + + pub fn shutdown(self: *Worker, io: std.Io) void { + self.should_stop.store(true, .release); + self.group.await(io) catch {}; + } + + pub fn isRunning(self: *const Worker) bool { + return !self.should_stop.load(.acquire); + } + + pub fn spawn( + self: *Worker, + io: std.Io, + comptime runFn: anytype, + args: std.meta.ArgsTuple(@TypeOf(runFn)), + ) !void { + try self.group.concurrent(io, runFn, args); + } +}; diff --git a/zml/io/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel similarity index 51% rename from zml/io/BUILD.bazel rename to bin/zml-smi/csv/BUILD.bazel index 8407a5c6..a0cb3422 100644 --- a/zml/io/BUILD.bazel +++ b/bin/zml-smi/csv/BUILD.bazel @@ -1,14 +1,11 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "io", - srcs = [ - "vfs.zig", - ], - main = "io.zig", + name = "csv", + import_name = "zml-smi/csv", + main = "csv.zig", visibility = ["//visibility:public"], deps = [ - "//stdx", - "//zml/io/vfs", + "//bin/zml-smi/info", ], ) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig new file mode 100644 index 00000000..dbd80dd9 --- /dev/null +++ b/bin/zml-smi/csv/csv.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { + const info = @typeInfo(DeviceInfo).@"union"; + inline for (info.field_names, info.field_types) |type_name, DeviceType| { + const tag = @field(smi_info.Target, type_name); + var header_printed = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + tag => |*db| { + if (!header_printed) { + try writer.writeAll("index,type"); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll("," ++ field_name); + } + try writer.writeAll("\n"); + + header_printed = true; + } + + const val = db.front().*; + + try writer.print("{d},{s}", .{ i, type_name }); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll(","); + try writeValue(writer, @field(val, field_name)); + } + try writer.writeAll("\n"); + }, + else => {}, + } + } + } +} + +fn writeValue(writer: *std.Io.Writer, field: anytype) !void { + if (@typeInfo(@TypeOf(field)) == .optional) { + if (field) |v| return writeValue(writer, v); + } else switch (@typeInfo(@TypeOf(field))) { + .pointer => return writer.writeAll(field), + else => return writer.print("{any}", .{field}), + } +} diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel new file mode 100644 index 00000000..d035b161 --- /dev/null +++ b/bin/zml-smi/info/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "info", + import_name = "zml-smi/info", + main = "info.zig", + srcs = [ + "device_info.zig", + "host_info.zig", + "poll_metrics.zig", + "process_info.zig", + ], + visibility = ["//visibility:public"], + deps = ["//bin/zml-smi/utils:double_buffer"], +) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig new file mode 100644 index 00000000..4b01c728 --- /dev/null +++ b/bin/zml-smi/info/device_info.zig @@ -0,0 +1,95 @@ +const std = @import("std"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const Target = enum { + cuda, + rocm, + oneapi, + neuron, + tpu, +}; + +pub const DeviceInfo = union(Target) { + cuda: DoubleBuffer(GpuInfo), + rocm: DoubleBuffer(GpuInfo), + oneapi: DoubleBuffer(GpuInfo), + neuron: DoubleBuffer(NeuronInfo), + tpu: DoubleBuffer(TpuInfo), + + pub fn isRemote(self: *const DeviceInfo) bool { + switch (self.*) { + inline else => |*db| return db.front().remote, + } + } +}; + +pub const GpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + cuda_driver_version: ?[]const u8 = null, + + // Utilization + util_percent: ?u64 = null, + encoder_util_percent: ?u64 = null, + decoder_util_percent: ?u64 = null, + + // Power + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + // Thermal + temperature: ?u64 = null, + fan_speed_percent: ?u64 = null, + + // Clocks + clock_graphics_mhz: ?u64 = null, + clock_sm_mhz: ?u64 = null, + clock_soc_mhz: ?u64 = null, + clock_mem_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + clock_mem_max_mhz: ?u64 = null, + + // Memory + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + mem_bus_width: ?u64 = null, + + // PCIe + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + pcie_bandwidth_mbps: ?u64 = null, + pcie_link_gen: ?u64 = null, + pcie_link_width: ?u64 = null, +}; + +pub const NeuronInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + + // Neuron per-core HBM breakdown (bytes) + nc_tensors: ?u64 = null, + nc_constants: ?u64 = null, + nc_model_code: ?u64 = null, + nc_shared_scratchpad: ?u64 = null, + nc_nonshared_scratchpad: ?u64 = null, + nc_runtime: ?u64 = null, + nc_driver: ?u64 = null, + nc_dma_rings: ?u64 = null, + nc_collectives: ?u64 = null, + nc_notifications: ?u64 = null, + nc_uncategorized: ?u64 = null, +}; + +pub const TpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, +}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig new file mode 100644 index 00000000..f33f6fd8 --- /dev/null +++ b/bin/zml-smi/info/host_info.zig @@ -0,0 +1,16 @@ +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const HostInfo = DoubleBuffer(HostData); + +pub const HostData = struct { + hostname: ?[]const u8 = null, + kernel: ?[]const u8 = null, + cpu_name: ?[]const u8 = null, + cpu_cores: ?u64 = null, + mem_total_kib: ?u64 = null, + mem_available_kib: ?u64 = null, + load_1: ?f32 = null, + load_5: ?f32 = null, + load_15: ?f32 = null, + uptime_seconds: ?u64 = null, +}; diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig new file mode 100644 index 00000000..42e1b1b3 --- /dev/null +++ b/bin/zml-smi/info/info.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub const device_info = @import("device_info.zig"); +pub const host_info = @import("host_info.zig"); +pub const poll_metrics = @import("poll_metrics.zig"); +pub const process_info = @import("process_info.zig"); + +pub const Target = device_info.Target; +pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig new file mode 100644 index 00000000..62ed94a7 --- /dev/null +++ b/bin/zml-smi/info/poll_metrics.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { + inline for (table) |m| { + @field(dest, m.field) = m.query(src) catch null; + } +} + +pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { + return struct { + fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { + if (arena) |a| _ = a.reset(.retain_capacity); + + const back = db.back(); + back.* = db.front().*; + apply(table, back, dev); + db.swap(); + } + }.f; +} diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig new file mode 100644 index 00000000..73504656 --- /dev/null +++ b/bin/zml-smi/info/process_info.zig @@ -0,0 +1,12 @@ +pub const ProcessInfo = struct { + pid: u32 = 0, + device_idx: u16 = 0, + dev_util_percent: ?u16 = null, + dev_mem_kib: ?u64 = null, + uid: u32 = 0, + username: []const u8 = "", + comm: []const u8 = "", + cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% + rss_kib: u64 = 0, + remote: bool = false, +}; diff --git a/bin/zml-smi/json/BUILD.bazel b/bin/zml-smi/json/BUILD.bazel new file mode 100644 index 00000000..decc8ca1 --- /dev/null +++ b/bin/zml-smi/json/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "json", + import_name = "zml-smi/json", + main = "json.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig new file mode 100644 index 00000000..c32a16cf --- /dev/null +++ b/bin/zml-smi/json/json.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { + var jw: std.json.Stringify = .{ .writer = writer }; + try jw.write(Response{ .devices = devices, .processes = processes }); + try writer.writeAll("\n"); +} + +const Response = struct { + devices: []const *DeviceInfo, + processes: []const ProcessInfo, +}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig new file mode 100644 index 00000000..f066dd34 --- /dev/null +++ b/bin/zml-smi/main.zig @@ -0,0 +1,241 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +pub const std_options: std.Options = .{ + .log_scope_levels = &.{ + .{ .scope = .vaxis, .level = .warn }, + }, +}; + +const Collector = @import("zml-smi/collector").Collector; +const smi_info = @import("zml-smi/info"); +const HostInfo = smi_info.host_info.HostInfo; +const platform = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos") +else + @import("zml-smi/platforms/linux"); +const ProcessEnricher = platform.process.ProcessEnricher; +const host = platform.metrics; +const c = @import("c"); +const csv = @import("zml-smi/csv"); +const json = @import("zml-smi/json"); +const api = @import("zml-smi/api"); +const prometheus = @import("zml-smi/prometheus"); +const sysfs = @import("zml-smi/sysfs"); +const smi_tui = @import("zml-smi/tui"); +const tui = smi_tui.top; +const static_print = smi_tui.print; +const data = smi_tui.data; + +const CliArgs = struct { + top: bool = false, + csv: bool = false, + json: bool = false, + api: bool = false, + api_port: u16 = 9090, + remotes: ?[]const u8 = null, + prometheus_listen: ?[]const u8 = null, + tui_refresh_rate: u16 = 100, + poll_interval: u16 = 500, + + pub const help = + \\ zml-smi [--top] [--csv] [--json] [--prometheus-listen HOST:PORT] + \\ [--tui-refresh-rate MS] [--poll-interval MS] + \\ + \\ --top Interactive TUI mode + \\ --csv Output device metrics as CSV + \\ --json Output device metrics as JSON + \\ --prometheus-listen Expose metrics as Prometheus endpoint on HOST:PORT + \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) + \\ --poll-interval Device polling interval in ms (default: 500) + \\ + ; + + // [--api] [--port PORT] [--remotes HOST,...] + // \\ --api Expose device metrics as HTTP JSON endpoint + // \\ --api-port API server port (default: 9090) + // \\ --remotes Add devices from remote hosts (comma-separated URLs) +}; + +const device_backends = switch (builtin.os.tag) { + .macos => .{}, + .linux => switch (builtin.cpu.arch) { + .x86_64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .oneapi, @import("zml-smi/platforms/oneapi") }, + .{ .neuron, @import("zml-smi/platforms/neuron") }, + .{ .tpu, @import("zml-smi/platforms/tpu") }, + }, + .aarch64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + }, + else => unreachable, + }, + else => unreachable, +}; + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const arena = init.arena.allocator(); + const io = init.io; + const args = stdx.flags.parse(init.minimal.args, CliArgs); + + const targets = detect(gpa, io); + + var collector: Collector = .init(arena, gpa, io, .{ + .poll_interval_ms = args.poll_interval, + .poll_only = !args.top and args.prometheus_listen == null and !args.api, + }); + defer collector.deinit(); + + var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; + try host.init(&collector, &host_info); + + var enricher: ProcessEnricher = try .init(gpa, io); + defer enricher.deinit(); + + if (args.api) { + std.log.info("serving api metrics on port {d}", .{args.api_port}); + } + + if (args.prometheus_listen) |listen| { + std.log.info("serving prometheus metrics on {s}", .{listen}); + } + + inline for (device_backends) |entry| { + const target, const backend = entry; + if (targets.contains(target)) { + backend.start(&collector) catch |err| { + std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); + }; + } + } + + var api_group: std.Io.Group = .init; + + if (args.remotes) |hosts| { + try api.addRemotes(&collector, hosts); + } + + var state = try data.SystemState.init(.{ + .devices = collector.device_infos.items, + .host = &host_info, + .targets = targets, + .tui_refresh_rate = args.tui_refresh_rate, + .process_lists = collector.process_lists.items, + .enricher = &enricher, + .gpa = gpa, + .arena = arena, + .io = io, + }); + defer state.deinit(arena); + + if (args.prometheus_listen) |listen| { + try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); + } + + if (args.api) { + try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); + } + + if (args.prometheus_listen != null or args.api) { + try api_group.await(io); + } else if (args.csv) { + var csv_buf: [4096]u8 = undefined; + var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); + + try csv.write(&csv_writer.interface, collector.device_infos.items); + try csv_writer.flush(); + } else if (args.json) { + var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; + defer procs.deinit(gpa); + for (collector.process_lists.items) |pl| { + try procs.appendSlice(gpa, pl.front().items); + } + enricher.enrich(io, procs.items); + + var json_buf: [4096]u8 = undefined; + var json_writer = std.Io.File.stdout().writer(io, &json_buf); + + try json.write(&json_writer.interface, collector.device_infos.items, procs.items); + try json_writer.flush(); + } else if (args.top) { + try tui.run(gpa, io, init.environ_map, &state); + } else { + try static_print.run(arena, io, init.environ_map, &state); + } +} + +fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { + if (comptime builtin.os.tag == .macos) { + return .{}; + } + + var targets: smi_info.Targets = .{}; + + switch (builtin.os.tag) { + .macos => {}, + .linux => { + switch (builtin.cpu.arch) { + .x86_64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } + + if (hasDrmRenderDevice(allocator, io, "0x8086")) { + targets.insert(.oneapi); + } + + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + }, + .aarch64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + }, + else => unreachable, + } + }, + else => unreachable, + } + + return targets; +} + +fn hasDevice(io: std.Io, path: []const u8) bool { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + return true; +} + +fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; + const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; + defer allocator.free(vendor); + if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { + return true; + } + } + + return false; +} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel new file mode 100644 index 00000000..dac6b935 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "amdsmi", + srcs = [ + "amdsmi.zig", + "process.zig", + ], + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//stdx", + "@amdrocm-amdsmi//:amdsmi", + ], +) diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig new file mode 100644 index 00000000..2d89bc7c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -0,0 +1,295 @@ +const std = @import("std"); +const c = @import("c"); +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const AmdSmi = @This(); + +pub const Error = ReturnError || error{AmdSmiUnavailable}; +pub const Handle = c.amdsmi_processor_handle; +pub const ProcInfo = c.amdsmi_proc_info_t; + +lib: Fns, +gpu_handles: []c.amdsmi_processor_handle, + +const Fns = struct { + amdsmi_init: *const @TypeOf(c.amdsmi_init), + amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), + amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), + amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), + amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), + amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), + amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), + amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), + amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), + amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), + amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), + amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), + amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), + amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), + amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), + amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), +}; + +pub fn init(allocator: std.mem.Allocator) !AmdSmi { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); + + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); + return error.AmdSmiUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; + + try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); + + var socket_count: u32 = 0; + try check(fns.amdsmi_get_socket_handles(&socket_count, null)); + + const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); + defer allocator.free(sockets); + try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); + + var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; + defer handle_list.deinit(allocator); + + for (sockets[0..socket_count]) |socket| { + var proc_count: u32 = 0; + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); + + const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); + defer allocator.free(procs); + + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); + + for (procs[0..proc_count]) |proc| { + try handle_list.append(allocator, proc); + } + } + + return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; +} + +pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { + if (device_id >= self.gpu_handles.len) { + return error.not_found; + } + return self.gpu_handles[device_id]; +} + +pub fn deviceCount(self: AmdSmi) u32 { + return @intCast(self.gpu_handles.len); +} + +pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; + +pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { + var info: c.amdsmi_asic_info_t = undefined; + try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); + @memcpy(buf, &info.market_name); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { + var metrics: c.amdsmi_gpu_metrics_t = undefined; + try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); + const unsupported16 = std.math.maxInt(u16); + if (metrics.current_socket_power != unsupported16) { + return metrics.current_socket_power; + } + if (metrics.average_socket_power != unsupported16) { + return metrics.average_socket_power; + } + return error.not_supported; +} + +pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_power_info_t = undefined; + try check(self.lib.amdsmi_get_power_info(handle, &info)); + return info.power_limit; +} + +pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { + var temp: i64 = 0; + try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); + return temp; +} + +pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { + var speed: i64 = 0; + try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); + return speed; +} + +pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { + var usage: c.amdsmi_engine_usage_t = undefined; + try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); + return usage.gfx_activity; +} + +pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.clk; +} + +pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.max_clk; +} + +pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.clk; +} + +pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.max_clk; +} + +pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); + return info.clk; +} + +pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { + var total: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); + return total; +} + +pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { + var used: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); + return used; +} + +pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return @intCast(info.pcie_metric.pcie_width); +} + +pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_metric.pcie_bandwidth; +} + +pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_static.pcie_interface_version; +} + +pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { + var bdf_id: u64 = 0; + try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); + return bdf_id; +} + +pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; + +pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { + var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); + try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); + @memcpy(buf, &info.driver_version); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { + var count: u32 = 0; + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); + if (count == 0) { + return &.{}; + } + const procs = try allocator.alloc(ProcInfo, count); + errdefer allocator.free(procs); + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); + return procs; +} + +// Error handling + +pub const ReturnError = error{ + inval, + not_supported, + not_yet_implemented, + fail_load_module, + fail_load_symbol, + drm_error, + api_failed, + timeout, + retry, + no_perm, + interrupt, + io_error, + address_fault, + file_error, + out_of_resources, + internal_exception, + input_out_of_bounds, + init_error, + refcount_overflow, + directory_not_found, + busy, + not_found, + not_init, + no_slot, + driver_not_loaded, + no_data, + insufficient_size, + unexpected_size, + unexpected_data, + unknown_error, +}; + +fn check(ret: c.amdsmi_status_t) Error!void { + if (ret == c.AMDSMI_STATUS_SUCCESS) { + return; + } + + return switch (ret) { + c.AMDSMI_STATUS_INVAL => error.inval, + c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, + c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, + c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, + c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, + c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, + c.AMDSMI_STATUS_API_FAILED => error.api_failed, + c.AMDSMI_STATUS_TIMEOUT => error.timeout, + c.AMDSMI_STATUS_RETRY => error.retry, + c.AMDSMI_STATUS_NO_PERM => error.no_perm, + c.AMDSMI_STATUS_INTERRUPT => error.interrupt, + c.AMDSMI_STATUS_IO => error.io_error, + c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, + c.AMDSMI_STATUS_FILE_ERROR => error.file_error, + c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, + c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, + c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, + c.AMDSMI_STATUS_INIT_ERROR => error.init_error, + c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, + c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, + c.AMDSMI_STATUS_BUSY => error.busy, + c.AMDSMI_STATUS_NOT_FOUND => error.not_found, + c.AMDSMI_STATUS_NOT_INIT => error.not_init, + c.AMDSMI_STATUS_NO_SLOT => error.no_slot, + c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, + c.AMDSMI_STATUS_NO_DATA => error.no_data, + c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, + c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, + c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, + else => error.unknown_error, + }; +} diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig new file mode 100644 index 00000000..8aa0487c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const AmdSmi = @import("amdsmi.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const amdsmi = try collector.arena.create(AmdSmi); + amdsmi.* = try AmdSmi.init(collector.arena); + + const count = amdsmi.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const dev = Device.open(amdsmi, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); + try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, amdsmi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + amdsmi: *const AmdSmi, + handle: AmdSmi.Handle, + + pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { + return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.name_buf_len]u8 = undefined; + const slice = try self.amdsmi.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; + const slice = try self.amdsmi.driverVersion(self.handle, &buf); + return try arena.dupe(u8, slice); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + const pw = try self.amdsmi.powerUsage(self.handle); + return @as(u64, pw) * 1000; + } + + pub fn powerLimit(self: Device) !u64 { + const limit = try self.amdsmi.powerLimit(self.handle); + if (limit == std.math.maxInt(u32)) { + return error.not_supported; + } + + return @as(u64, limit) / 1000; + } + + // Thermal + pub fn temperature(self: Device) !u64 { + const temp = try self.amdsmi.temperature(self.handle); + return @intCast(temp); + } + + pub fn fanSpeed(self: Device) !u64 { + const speed = try self.amdsmi.fanSpeed(self.handle); + return @intCast(@divTrunc(speed * 100, 255)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return try self.amdsmi.gpuUtil(self.handle); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return try self.amdsmi.clockGraphics(self.handle); + } + + pub fn clockSoc(self: Device) !u64 { + return try self.amdsmi.clockSoc(self.handle); + } + + pub fn clockMem(self: Device) !u64 { + return try self.amdsmi.clockMem(self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return try self.amdsmi.maxClockGraphics(self.handle); + } + + pub fn maxClockMem(self: Device) !u64 { + return try self.amdsmi.maxClockMem(self.handle); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.amdsmi.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.amdsmi.memTotal(self.handle); + } + + // PCIe + pub fn pcieBandwidth(self: Device) !u64 { + const bw = try self.amdsmi.pcieBandwidth(self.handle); + if (bw == std.math.maxInt(u32)) { + return error.not_supported; + } + + return bw; + } + + pub fn pcieLinkGen(self: Device) !u64 { + return try self.amdsmi.pcieLinkGen(self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return try self.amdsmi.pcieWidth(self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig new file mode 100644 index 00000000..8d1fdd01 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -0,0 +1,139 @@ +const std = @import("std"); +const c = std.c; +const AmdSmi = @import("amdsmi.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +const bdf_len = "0000:00:00.0".len; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { + const device_count = amdsmi.deviceCount(); + const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); + + for (pci_slots, 0..) |*slot, i| { + const handle = amdsmi.handleByIndex(@intCast(i)) catch { + slot.* = null; + continue; + }; + + slot.* = formatBdf(amdsmi.bdfId(handle) catch { + slot.* = null; + continue; + }); + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; + + const procs = blk: { + const saved = c.dup(c.STDERR_FILENO); + const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + + if (devnull >= 0) { + if (saved >= 0) { + _ = c.dup2(devnull, c.STDERR_FILENO); + } + + _ = c.close(devnull); + } + + defer if (saved >= 0) { + _ = c.dup2(saved, c.STDERR_FILENO); + _ = c.close(saved); + }; + + break :blk amdsmi.processList(allocator, handle) catch continue; + }; + defer { + if (procs.len > 0) { + allocator.free(procs); + } + } + + const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; + + for (procs) |proc| { + back.append(allocator, .{ + .pid = proc.pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), + }) catch break; + } + } + + list.swap(); +} + +/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. +/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. +fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; + + var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; + defer dir.close(io); + + var total_vram_kib: u64 = 0; + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; + + total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); + } + + return total_vram_kib; +} + +fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { + var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var found_pci = false; + var vram_kib: ?u64 = null; + + while (true) { + const line = reader.interface.takeDelimiterExclusive('\n') catch break; + reader.interface.toss(1); + + if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; + + // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) + inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { + if (std.mem.startsWith(u8, line, key)) { + const rest = line[key.len..]; + const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; + vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; + } + } + + if (found_pci and vram_kib != null) return vram_kib.?; + } + + return 0; +} + +fn formatBdf(bdf_id: u64) [bdf_len]u8 { + var buf: [bdf_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + (bdf_id >> 16) & 0xFFFF, + (bdf_id >> 8) & 0xFF, + (bdf_id >> 3) & 0x1F, + bdf_id & 0x7, + }) catch unreachable; + + return buf; +} diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel new file mode 100644 index 00000000..57bb96d7 --- /dev/null +++ b/bin/zml-smi/platforms/linux/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "linux", + import_name = "zml-smi/platforms/linux", + main = "linux.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/linux/linux.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig new file mode 100644 index 00000000..5a69663a --- /dev/null +++ b/bin/zml-smi/platforms/linux/metrics.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const HostData = host_info.HostData; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{ .io = collector.io, .arena = poll_arena }; + + // read once metrics + var initial = info.front().*; + initial.hostname = host.hostname(collector.arena) catch null; + initial.kernel = host.kernel(collector.arena) catch null; + initial.cpu_name = host.cpuName(collector.arena) catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + + pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); + } + + pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); + } + + pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); + } + + pub fn cpuCores(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); + const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; + + return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; + } + + pub fn memTotal(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); + } + + pub fn memAvailable(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); + } + + fn parseLoadAvg(self: Host, comptime n: usize) !f32 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); + var iter = std.mem.splitScalar(u8, data, ' '); + + var i: usize = 0; + while (iter.next()) |tok| : (i += 1) { + if (i == n) return std.fmt.parseFloat(f32, tok); + } + + return error.NotFound; + } + + pub fn load1(self: Host) !f32 { + return self.parseLoadAvg(0); + } + + pub fn load5(self: Host) !f32 { + return self.parseLoadAvg(1); + } + + pub fn load15(self: Host) !f32 { + return self.parseLoadAvg(2); + } + + pub fn uptime(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); + const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; + + return std.fmt.parseInt(u64, data[0..dot], 10); + } +}; + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig new file mode 100644 index 00000000..c57d1725 --- /dev/null +++ b/bin/zml-smi/platforms/linux/process.zig @@ -0,0 +1,118 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const pi = @import("zml-smi/info").process_info; + +const ProcessInfo = pi.ProcessInfo; + +pub const ProcessEnricher = struct { + gpa: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, + prev_total_ticks: u64 = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + + const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); + var enricher: ProcessEnricher = .{ + .gpa = gpa, + .arena = arena, + .prev_total_ticks = prev_total_ticks, + .prev_ticks = .{}, + }; + + try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); + + return enricher; + } + + pub fn deinit(self: *ProcessEnricher) void { + self.prev_ticks.deinit(self.gpa); + self.arena.deinit(); + } + + pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { + _ = self.arena.reset(.retain_capacity); + const alloc = self.arena.allocator(); + + const curr_total = readTotalCpuTicks(alloc, io); + const delta_total = curr_total -| self.prev_total_ticks; + + var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; + curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; + + for (procs) |*info| { + if (info.remote) continue; + // Read Uid, VmRSS from /proc/{pid}/status + const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; + const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); + info.uid = uid; + info.username = lookupUsername(alloc, uid) catch continue; + info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; + + // Read cmdline + const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; + info.comm = readCmdline(alloc, io, cmdline_path) catch continue; + + // CPU% delta + const ticks = readProcTicks(alloc, io, info.pid) orelse continue; + if (delta_total > 0) { + if (self.prev_ticks.get(info.pid)) |prev| { + const delta_proc = ticks -| prev; + info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); + } + } + + curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; + } + + self.prev_ticks.deinit(self.gpa); + self.prev_ticks = curr_ticks; + self.prev_total_ticks = curr_total; + } +}; + +fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { + const pw = std.c.getpwuid(uid) orelse return error.NoUser; + const name = std.mem.span(pw.name orelse return error.NoName); + + return try allocator.dupe(u8, name); +} + +/// Extract utime + stime from /proc/{pid}/stat. +fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { + const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; + const data = sysfs.readFirstLine(allocator, io, path) catch return null; + + // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 + const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; + + return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); +} + +fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + const data = try sysfs.readFirstLine(allocator, io, path); + + for (data) |*c| { + if (c.* == 0) { + c.* = ' '; + } + } + + return std.mem.trimEnd(u8, data, &std.ascii.whitespace); +} + +fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { + const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; + + var total: u64 = 0; + var iter = std.mem.tokenizeAny(u8, data, " \t"); + + _ = iter.next(); // skip "cpu" + while (iter.next()) |tok| { + total += std.fmt.parseInt(u64, tok, 10) catch 0; + } + + return total; +} diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel new file mode 100644 index 00000000..a92e298e --- /dev/null +++ b/bin/zml-smi/platforms/macos/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "macos", + import_name = "zml-smi/platforms/macos", + main = "macos.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/macos/macos.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig new file mode 100644 index 00000000..616168cd --- /dev/null +++ b/bin/zml-smi/platforms/macos/metrics.zig @@ -0,0 +1,161 @@ +const std = @import("std"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +const c = std.c; + +const vm_statistics64 = extern struct { + free_count: c_uint, + active_count: c_uint, + inactive_count: c_uint, + wire_count: c_uint, + zero_fill_count: u64, + reactivations: u64, + pageins: u64, + pageouts: u64, + faults: u64, + cow_faults: u64, + lookups: u64, + hits: u64, + purges: u64, + purgeable_count: c_uint, + speculative_count: c_uint, + decompressions: u64, + compressions: u64, + swapins: u64, + swapouts: u64, + compressor_page_count: c_uint, + throttled_count: c_uint, + external_page_count: c_uint, + internal_page_count: c_uint, + total_uncompressed_pages_in_compressor: u64, +}; + +const HOST_VM_INFO64: c_int = 4; +const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); +const KERN_SUCCESS: c_int = 0; + +extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; +extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{}; + + var initial = info.front().*; + initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; + initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; + initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + pub fn cpuCores(_: Host) !u64 { + return @intCast(try sysctlU32("hw.logicalcpu")); + } + + pub fn memTotal(_: Host) !u64 { + return (try sysctlU64("hw.memsize")) / 1024; + } + + pub fn memAvailable(_: Host) !u64 { + var vm_stat: vm_statistics64 = undefined; + var count: c_uint = HOST_VM_INFO64_COUNT; + + if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { + return error.HostStatsFailed; + } + + var page_size: u64 = 0; + if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { + return error.PageSizeFailed; + } + + const available_pages: u64 = @as(u64, vm_stat.free_count) + + @as(u64, vm_stat.inactive_count) + + @as(u64, vm_stat.purgeable_count); + + return (available_pages * page_size) / 1024; + } + + fn loadAvg() ![3]f64 { + var avg: [3]f64 = undefined; + if (getloadavg(&avg, 3) != 3) { + return error.LoadAvgFailed; + } + + return avg; + } + + pub fn load1(_: Host) !f32 { + return @floatCast((try loadAvg())[0]); + } + + pub fn load5(_: Host) !f32 { + return @floatCast((try loadAvg())[1]); + } + + pub fn load15(_: Host) !f32 { + return @floatCast((try loadAvg())[2]); + } + + pub fn uptime(_: Host) !u64 { + var boottime: c.timeval = undefined; + var len: usize = @sizeOf(c.timeval); + if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + var now: c.timeval = undefined; + _ = c.gettimeofday(&now, null); + + return @intCast(now.sec - boottime.sec); + } +}; + +fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { + var buf: [256]u8 = undefined; + var len: usize = buf.len; + if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); +} + +fn sysctlU64(name: [*:0]const u8) !u64 { + var value: u64 = 0; + var len: usize = @sizeOf(u64); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +fn sysctlU32(name: [*:0]const u8) !u32 { + var value: u32 = 0; + var len: usize = @sizeOf(u32); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig new file mode 100644 index 00000000..a48e9831 --- /dev/null +++ b/bin/zml-smi/platforms/macos/process.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessEnricher = struct { + pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { + return .{}; + } + + pub fn deinit(_: *ProcessEnricher) void {} + + pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} +}; diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel new file mode 100644 index 00000000..c7129ac7 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "neuron", + import_name = "zml-smi/platforms/neuron", + main = "metrics.zig", + srcs = [ + "nrt.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//bin/zml-smi/utils:sysfs", + "//platforms/neuron:libnrt_headers", + "//stdx", + ], +) diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig new file mode 100644 index 00000000..0b5a5ddd --- /dev/null +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -0,0 +1,201 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const NeuronInfo = device_info.NeuronInfo; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Nrt = @import("nrt.zig"); +const process = @import("process.zig"); + +const base_path = "/sys/devices/virtual/neuron_device"; + +pub const UtilAtomic = std.atomic.Value(u64); + +pub fn start(collector: *Collector) !void { + const nrt = try collector.arena.create(Nrt); + nrt.* = try Nrt.init(collector.arena, collector.io); + + if (nrt.handles.len == 0) { + return; + } + + const total_nc = nrt.totalNcCount() catch return; + const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); + + const total_cores = @as(usize, nrt.handles.len) * nc_per_device; + const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); + for (util_buf) |*u| u.* = .init(0); + + const GiB: u64 = 1024 * 1024 * 1024; + var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + var ver_buf: [Nrt.version_buf_len]u8 = undefined; + const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| + collector.arena.dupe(u8, v) catch null + else |_| + null; + + var flat_core: usize = 0; + for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { + const device_type = Nrt.deviceType(dev_handle); + const mem_per_device: u64 = switch (device_type) { + .inf1 => 8 * GiB, + .inf2_trn1 => 32 * GiB, + .trn2 => 96 * GiB, + .trn3 => 144 * GiB, + else => 0, + }; + + for (0..nc_per_device) |ci| { + const poll_arena = try collector.createPollArena(); + + const dev: Device = .{ + .io = collector.io, + .arena = poll_arena, + .device_idx = @intCast(device_idx), + .core_idx = @intCast(ci), + .mem_per_device = mem_per_device, + .nc_per_device = nc_per_device, + .util_src = &util_buf[flat_core], + }; + + const initial: NeuronInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = nrt_ver, + }; + const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); + try neuron_infos.append(collector.arena, info); + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); + flat_core += 1; + } + } + + if (neuron_infos.items.len > 0) { + const processes = try collector.createProcessList(); + const warmup = collector.poll_only; + try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); + + if (warmup) { + for (neuron_infos.items, util_buf) |info, *u| { + const back = info.neuron.back(); + back.* = info.neuron.front().*; + back.util_percent = u.load(.acquire); + info.neuron.swap(); + } + } + } +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); + +const Device = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + device_idx: u32, + core_idx: u32, + mem_per_device: u64 = 0, + nc_per_device: u32 = 1, + util_src: *const UtilAtomic, + + pub fn utilPercent(self: Device) !u64 { + return self.util_src.load(.acquire); + } + + fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); + } + + fn corePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); + } + + fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); + } + + fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { + return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); + } + + pub fn memTotal(self: Device) !u64 { + if (self.mem_per_device == 0) { + return error.NrtUnavailable; + } + return self.mem_per_device; + } + + // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. + pub fn memUsed(self: Device) !u64 { + var total: u64 = 0; + for (0..self.nc_per_device) |ci| { + const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); + total += try sysfs.readInt(self.arena.allocator(), self.io, path); + } + + return total; + } + + pub fn tensors(self: Device) !u64 { + return self.readCoreMem("tensors"); + } + + pub fn constants(self: Device) !u64 { + return self.readCoreMem("constants"); + } + + pub fn modelCode(self: Device) !u64 { + return self.readCoreMem("model_code"); + } + + pub fn sharedScratchpad(self: Device) !u64 { + return self.readCoreMem("model_shared_scratchpad"); + } + + pub fn nonsharedScratchpad(self: Device) !u64 { + return self.readCoreMem("nonshared_scratchpad"); + } + + pub fn runtime(self: Device) !u64 { + return self.readCoreMem("runtime_memory"); + } + + pub fn driver(self: Device) !u64 { + return self.readCoreMem("driver_memory"); + } + + pub fn dmaRings(self: Device) !u64 { + return self.readCoreMem("dma_rings"); + } + + pub fn collectives(self: Device) !u64 { + return self.readCoreMem("collectives"); + } + + pub fn notifications(self: Device) !u64 { + return self.readCoreMem("notifications"); + } + + pub fn uncategorized(self: Device) !u64 { + return self.readCoreMem("uncategorized"); + } +}; + +const metrics = .{ + .{ .field = "util_percent", .query = Device.utilPercent }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "nc_tensors", .query = Device.tensors }, + .{ .field = "nc_constants", .query = Device.constants }, + .{ .field = "nc_model_code", .query = Device.modelCode }, + .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, + .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, + .{ .field = "nc_runtime", .query = Device.runtime }, + .{ .field = "nc_driver", .query = Device.driver }, + .{ .field = "nc_dma_rings", .query = Device.dmaRings }, + .{ .field = "nc_collectives", .query = Device.collectives }, + .{ .field = "nc_notifications", .query = Device.notifications }, + .{ .field = "nc_uncategorized", .query = Device.uncategorized }, +}; diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig new file mode 100644 index 00000000..cd77b3c0 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -0,0 +1,230 @@ +const std = @import("std"); +const c = @import("c"); +const elf = std.elf; +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const Nrt = @This(); + +pub const Error = error{ nrt_error, NrtUnavailable }; +pub const AppInfo = c.struct_neuron_app_info; +pub const DeviceType = enum(c_int) { + inf1 = 1, + inf2_trn1 = 2, + trn2 = 3, + trn3 = 4, + _, +}; + +lib: Fns, +private_lib: PrivateFns, +handles: []const *c.ndl_device_t, +device_indexes: []const c_int, + +const Fns = struct { + nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), + nrt_get_version: *const @TypeOf(c.nrt_get_version), +}; + +const PrivateFns = struct { + nds_open: *const @TypeOf(c.nds_open), + ndl_available_devices: *const @TypeOf(c.ndl_available_devices), + ndl_open_device: *const @TypeOf(c.ndl_open_device), + ndl_close_device: *const @TypeOf(c.ndl_close_device), + ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), + nds_close: *const @TypeOf(c.nds_close), + nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse { + std.log.err("neuron: sandbox path unavailable", .{}); + return error.NrtUnavailable; + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { + std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); + return error.NrtUnavailable; + }; + + const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); + return error.NrtUnavailable; + }; + var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; + + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; + const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { + std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); + return error.NrtUnavailable; + }; + + var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); + + var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; + var index_list: std.ArrayList(c_int) = .empty; + + for (dev_index_buf[0..count]) |device_idx| { + var dev: ?*c.ndl_device_t = null; + var t: c.struct_ndl_device_init_param = .{ + .initialize_device = false, + .num_dram_regions = 0, + .map_hbm = false, + .skip_copy_bufs = true, + }; + + if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { + if (dev) |d| { + errdefer _ = private_fns.ndl_close_device(d); + try handle_list.append(allocator, d); + try index_list.append(allocator, device_idx); + } + } + } + + return .{ + .lib = fns, + .private_lib = private_fns, + .handles = try handle_list.toOwnedSlice(allocator), + .device_indexes = try index_list.toOwnedSlice(allocator), + }; +} + +pub fn deviceCount(self: Nrt) u32 { + return @intCast(self.handles.len); +} + +pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { + var info: ?[*]AppInfo = null; + var count: usize = 0; + if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { + return error.nrt_error; + } + return .{ .ptr = info, .count = count }; +} + +pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { + var inst: ?*c.nds_instance_t = null; + if (self.private_lib.nds_open(dev, pid, &inst) != 0) { + return error.nrt_error; + } + return inst orelse error.nrt_error; +} + +pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { + _ = self.private_lib.nds_close(inst); +} + +pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { + var value: u64 = 0; + if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { + return error.nrt_error; + } + return value; +} + +pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; + +pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { + var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); + if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { + return error.nrt_error; + } + @memcpy(buf, &ver.rt_detail); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn totalNcCount(self: Nrt) Error!u32 { + var count: u32 = 0; + if (self.lib.nrt_get_total_nc_count(&count) != 0) { + return error.nrt_error; + } + return count; +} + +pub fn hbmSize(dev: *c.ndl_device_t) usize { + return dev.hbm_size; +} + +pub fn deviceType(dev: *c.ndl_device_t) DeviceType { + return @fromBackingInt(@intCast(dev.device_type)); +} + +fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { + const field_names = comptime std.meta.fieldNames(PrivateFns); + + const sym_addr = std.c.dlsym(handle, known_sym) orelse + return error.SymbolNotFound; + + var dl_info: DlInfo = undefined; + if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; + + const elf_base = @intFromPtr(dl_info.dli_fbase); + const elf_path = std.mem.span(dl_info.dli_fname); + + const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + const mmap = try std.posix.mmap( + null, + @intCast(stat.size), + .{ .READ = true }, + .{ .TYPE = .PRIVATE }, + file.handle, + 0, + ); + defer std.posix.munmap(mmap); + + var result: PrivateFns = undefined; + var remaining: usize = field_names.len; + + const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); + const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; + + for (shdrs) |sh| { + if (sh.type != .SYMTAB) { + continue; + } + + const strtab_sh = shdrs[sh.link]; + const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; + + for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { + if (s.value == 0) { + continue; + } + + const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); + + inline for (field_names) |field_name| { + if (std.mem.eql(u8, sym_name, field_name)) { + @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); + remaining -= 1; + } + } + + if (remaining == 0) { + return result; + } + } + } + + if (remaining > 0) { + return error.SymbolResolutionFailed; + } + return result; +} + +const DlInfo = extern struct { + dli_fname: [*:0]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: ?*anyopaque, +}; + +extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig new file mode 100644 index 00000000..2dbf363c --- /dev/null +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -0,0 +1,108 @@ +const std = @import("std"); +const c = @import("c"); +const Nrt = @import("nrt.zig"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; +const UtilAtomic = @import("metrics.zig").UtilAtomic; + +const DeltaState = struct { + prev_total_us: u64 = 0, + prev_timestamp: ?std.Io.Timestamp = null, +}; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { + if (nrt.handles.len == 0) return; + const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); + @memset(delta_states, .{}); + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; + if (warmup) { + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + } + + try collector.spawnPoll(pollOnce, args); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (nrt.handles, 0..) |handle, dev_i| { + const apps_result = nrt.allAppsInfo(handle) catch continue; + defer { + if (apps_result.ptr) |ptr| { + std.c.free(@ptrCast(ptr)); + } + } + + const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; + + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); + + for (apps) |*app| { + if (app.pid <= 0) { + continue; + } + + const nds = nrt.ndsOpen(handle, app.pid) catch continue; + defer nrt.ndsClose(nds); + + for (0..nc_per_device) |ci| { + const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; + total_us_per_core[ci] += time_in_use; + + if (time_in_use == 0) { + continue; + } + + const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); + + const pid = std.math.cast(u32, app.pid) orelse continue; + var info: pi.ProcessInfo = .{ + .pid = pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_mem_kib = @intCast(app.device_mem_size / 1024), + }; + + if (dev_idx < util_buf.len) { + info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); + } + + if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) + back.append(allocator, info) catch break; + } + } + + const now: std.Io.Timestamp = .now(io, .awake); + + for (0..nc_per_device) |ci| { + const dev_idx = dev_i * nc_per_device + ci; + if (dev_idx >= delta_states.len) { + break; + } + + const ds = &delta_states[dev_idx]; + const prev_us = ds.prev_total_us; + ds.prev_total_us = total_us_per_core[ci]; + + if (ds.prev_timestamp) |prev| { + const elapsed = prev.untilNow(io, .awake); + const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); + + if (elapsed_us > 0) { + const delta_us = total_us_per_core[ci] -| prev_us; + util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); + } + } + + ds.prev_timestamp = now; + } + } + + list.swap(); +} diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel new file mode 100644 index 00000000..0ab0fae1 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "nvml", + srcs = [ + "nvml.zig", + "process.zig", + ], + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + ] + select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], + }), +) diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig new file mode 100644 index 00000000..213dd253 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -0,0 +1,222 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const process = @import("process.zig"); + +const pcie_every: u8 = 4; + +const Slot = struct { + info: *DoubleBuffer(GpuInfo), + dev: Device, +}; + +const Ctx = struct { + gpa: std.mem.Allocator, + nvml: *const Nvml, + slots: []Slot, + processes: *process.ProcessDoubleBuffer, + last_seen_ts: []u64, + dev_offset: u16, + device_count: u32, + tick: u8 = 0, +}; + +pub fn start(collector: *Collector) !void { + const nvml = try collector.arena.create(Nvml); + nvml.* = try Nvml.init(); + + const count = try nvml.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); + for (0..count) |i| { + const dev = Device.open(nvml, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); + slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); + } + + const last_seen_ts = try collector.arena.alloc(u64, count); + @memset(last_seen_ts, 0); + + const ctx = try collector.arena.create(Ctx); + ctx.* = .{ + .gpa = collector.gpa, + .nvml = nvml, + .slots = slots.items, + .processes = try collector.createProcessList(), + .last_seen_ts = last_seen_ts, + .dev_offset = dev_offset, + .device_count = count, + }; + try collector.spawnPoll(pollOnce, .{ctx}); +} + +fn pollOnce(ctx: *Ctx) void { + const pcie = ctx.tick % pcie_every == 0; + ctx.tick +%= 1; + + for (ctx.slots) |slot| { + pollDevice(slot.info, slot.dev, pcie); + } + process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); +} + +fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { + const back = db.back(); + back.* = db.front().*; + poll_metrics.apply(metrics, back, dev); + if (pcie) poll_metrics.apply(pcie_metrics, back, dev); + db.swap(); +} + +const Device = struct { + nvml: *const Nvml, + handle: Nvml.Handle, + + pub fn open(nvml: *const Nvml, index: u32) !Device { + return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.name_buf_len]u8 = undefined; + const slice = try self.nvml.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.driver_version_buf_size]u8 = undefined; + const slice = try self.nvml.driverVersion(&buf); + + return try arena.dupe(u8, slice); + } + + fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + const v = try self.nvml.cudaDriverVersion(); + return try std.fmt.allocPrint(arena, "{}.{}", .{ + Nvml.cudaDriverVersionMajor(v), + Nvml.cudaDriverVersionMinor(v), + }); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + return @intCast(try self.nvml.powerUsage(self.handle)); + } + + pub fn powerLimit(self: Device) !u64 { + return @intCast(try self.nvml.powerLimit(self.handle)); + } + + // Thermal + pub fn temperature(self: Device) !u64 { + return @intCast(try self.nvml.temperature(self.handle)); + } + + pub fn fanSpeed(self: Device) !u64 { + return @intCast(try self.nvml.fanSpeed(self.handle)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return @intCast(try self.nvml.utilizationGpu(self.handle)); + } + + pub fn encoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.encoderUtil(self.handle)); + } + + pub fn decoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.decoderUtil(self.handle)); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.clockGraphics(self.handle)); + } + + pub fn clockSm(self: Device) !u64 { + return @intCast(try self.nvml.clockSm(self.handle)); + } + + pub fn clockMem(self: Device) !u64 { + return @intCast(try self.nvml.clockMem(self.handle)); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.maxClockGraphics(self.handle)); + } + + pub fn maxClockMem(self: Device) !u64 { + return @intCast(try self.nvml.maxClockMem(self.handle)); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.nvml.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.nvml.memTotal(self.handle); + } + + pub fn memBusWidth(self: Device) !u64 { + return @intCast(try self.nvml.memBusWidth(self.handle)); + } + + // PCIe + pub fn pcieTx(self: Device) !u64 { + return @intCast(try self.nvml.pcieTxKBps(self.handle)); + } + + pub fn pcieRx(self: Device) !u64 { + return @intCast(try self.nvml.pcieRxKBps(self.handle)); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieSpeed(self.handle)); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkGen(self.handle)); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkWidth(self.handle)); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_sm_mhz", .query = Device.clockSm }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_bus_width", .query = Device.memBusWidth }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; + +const pcie_metrics = .{ + .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, + .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, +}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig new file mode 100644 index 00000000..e2b36982 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/nvml.zig @@ -0,0 +1,349 @@ +const std = @import("std"); +const c = @import("c"); +const DynLib = @import("zml-smi/dynlib"); + +const Nvml = @This(); + +pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; +pub const Error = ReturnError || error{NvmlUnavailable}; +pub const Handle = c.nvmlDevice_t; +pub const ProcessInfo_t = c.nvmlProcessInfo_t; + +lib: Fns, + +const Fns = struct { + nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), + nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), + nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), + nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), + nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), + nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), + nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), + nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), + nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), + nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), + nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), + nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), + nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), + nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), + nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), + nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), + nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), + nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), + nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), + nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), + nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), + nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), + nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), + nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), +}; + +pub fn init() Error!Nvml { + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse + std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); + return error.NvmlUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; + try check(fns.nvmlInit_v2()); + return .{ .lib = fns }; +} + +pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { + var handle: c.nvmlDevice_t = undefined; + try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); + return handle; +} + +pub fn deviceCount(self: Nvml) Error!u32 { + var count: c_uint = 0; + try check(self.lib.nvmlDeviceGetCount_v2(&count)); + return @intCast(count); +} + +pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; + +pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var power_mw: c_uint = 0; + try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); + return power_mw; +} + +pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var temp: c_uint = 0; + try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); + return temp; +} + +pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c.nvmlUtilization_t = undefined; + try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); + return util.gpu; +} + +pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); + return clock; +} + +pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.total); +} + +pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.used); +} + +pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); + return speed; +} + +pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var limit: c_uint = 0; + try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); + return limit; +} + +pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); + return value; +} + +pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); + return value; +} + +pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var gen: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); + return gen; +} + +pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); + return width; +} + +pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); + return speed; +} + +pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn cudaDriverVersionMajor(v: c_int) c_int { + return @divTrunc(v, 1000); +} + +pub fn cudaDriverVersionMinor(v: c_int) c_int { + return @divTrunc(@mod(v, 1000), 10); +} + +pub fn cudaDriverVersion(self: Nvml) Error!c_int { + var version: c_int = 0; + try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); + return version; +} + +pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); + return width; +} + +const max_query = 4096; + +fn sizedCount(count: c_uint) ?c_uint { + if (count == 0 or count > max_query) return null; + return count; +} + +pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + + try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); + errdefer allocator.free(samples); + + try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); + return samples; +} + +// Error handling + +pub const ReturnError = error{ + error_uninitialized, + error_invalid_argument, + error_not_supported, + error_no_permission, + error_already_initialized, + error_not_found, + error_insufficient_size, + error_insufficient_power, + error_driver_not_loaded, + error_timeout, + error_irq_issue, + error_library_not_found, + error_function_not_found, + error_corrupted_inforom, + error_gpu_is_lost, + error_reset_required, + error_operating_system, + error_lib_rm_version_mismatch, + error_in_use, + error_memory, + error_no_data, + error_vgpu_ecc_not_supported, + error_insufficient_resources, + error_freq_not_supported, + error_argument_version_mismatch, + error_deprecated, + error_not_ready, + error_gpu_not_found, + error_invalid_state, + error_reset_type_not_supported, + error_unknown, +}; + +fn check(ret: c_uint) Error!void { + if (ret == c.NVML_SUCCESS) { + return; + } + return switch (ret) { + c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, + c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, + c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, + c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, + c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, + c.NVML_ERROR_NOT_FOUND => error.error_not_found, + c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, + c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, + c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, + c.NVML_ERROR_TIMEOUT => error.error_timeout, + c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, + c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, + c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, + c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, + c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, + c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, + c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, + c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, + c.NVML_ERROR_IN_USE => error.error_in_use, + c.NVML_ERROR_MEMORY => error.error_memory, + c.NVML_ERROR_NO_DATA => error.error_no_data, + c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, + c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, + c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, + c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, + c.NVML_ERROR_DEPRECATED => error.error_deprecated, + c.NVML_ERROR_NOT_READY => error.error_not_ready, + c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, + c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, + c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, + c.NVML_ERROR_UNKNOWN => error.error_unknown, + else => error.error_unknown, + }; +} diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig new file mode 100644 index 00000000..5e18cdcb --- /dev/null +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); + +pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; + const idx: u16 = @intCast(dev_idx + dev_offset); + + const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; + defer if (compute.len > 0) allocator.free(compute); + collectFromQuery(allocator, back, idx, compute); + + const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; + defer if (graphics.len > 0) allocator.free(graphics); + collectFromQuery(allocator, back, idx, graphics); + + const last_ts = last_seen_ts[dev_idx]; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; + defer if (utils.len > 0) allocator.free(utils); + + for (utils) |sample| { + if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { + continue; + } + + last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); + + for (back.items) |*entry| { + if (entry.pid == sample.pid and entry.device_idx == idx) { + entry.dev_util_percent = @intCast(sample.smUtil); + } + } + } + } + + list.swap(); +} + +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { + for (procs) |gp| { + back.append(allocator, .{ + .pid = gp.pid, + .device_idx = dev_idx, + .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), + }) catch return; + } +} diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..c0b27d84 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "oneapi", + import_name = "zml-smi/platforms/oneapi", + main = "metrics.zig", + srcs = [ + "monitor.zig", + "oneapi.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) + +zig_test( + name = "test", + deps = [":oneapi"], +) diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig new file mode 100644 index 00000000..ae2a86a3 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/metrics.zig @@ -0,0 +1,134 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const oneapi = try collector.arena.create(OneApi); + oneapi.* = try OneApi.init(collector.arena, collector.io); + + const count = oneapi.monitor.devices.len; + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const poll_arena = try collector.createPollArena(); + const dev = Device.open(oneapi, i, poll_arena) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .power_limit_mw = dev.powerLimit() catch null, + .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, + .mem_total_bytes = dev.memTotal() catch null, + .pcie_link_gen = dev.pcieLinkGen() catch null, + .pcie_link_width = dev.pcieLinkWidth() catch null, + .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, + }; + const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); + + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, oneapi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + oneapi: *OneApi, + arena: *std.heap.ArenaAllocator, + handle: OneApi.Handle, + + fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { + return .{ + .oneapi = oneapi, + .arena = arena, + .handle = try oneapi.handleByIndex(index), + }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.name(arena, self.handle); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.driverVersion(arena); + } + + pub fn powerUsage(self: Device) !u64 { + return self.oneapi.powerUsage(self.arena.allocator(), self.handle); + } + + pub fn temperature(self: Device) !u64 { + return self.oneapi.temperature(self.arena.allocator(), self.handle); + } + + pub fn fanSpeed(self: Device) !u64 { + return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); + } + + pub fn gpuUtil(self: Device) !u64 { + return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); + } + + pub fn encoderUtil(self: Device) !u64 { + return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); + } + + pub fn decoderUtil(self: Device) !u64 { + return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); + } + + pub fn powerLimit(self: Device) !u64 { + return self.oneapi.powerLimit(self.arena.allocator(), self.handle); + } + + pub fn clockGraphics(self: Device) !u64 { + return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); + } + + pub fn memUsed(self: Device) !u64 { + return self.oneapi.memUsed(self.arena.allocator(), self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.oneapi.memTotal(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig new file mode 100644 index 00000000..dd5f97a9 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -0,0 +1,877 @@ +const std = @import("std"); +const smi_sysfs = @import("zml-smi/sysfs"); +const Monitor = @This(); + +pub const bus_device_identifier_len = "0000:00:00.0".len; +pub const Handle = enum(u32) { _ }; + +const Device = struct { + render_idx: usize, + drm_path: []const u8, + dev_path: []const u8, + pcie_path: []const u8, + hwmon_path: ?[]const u8, + bus_device_identifier: ?[bus_device_identifier_len]u8, + activity_prev: ?EngineSample = null, + encoder_prev: ?EngineSample = null, + decoder_prev: ?EngineSample = null, + energy_prev: ?EnergySample = null, +}; + +const EnergySample = struct { + micro_joules: u64, + timestamp_ns: u64, +}; + +pub const EngineSample = struct { + engine_ns: u64, + timestamp_ns: u64, +}; + +const DeviceSample = struct { + mem_kib: u64 = 0, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessSample = struct { + pid: u32, + device_idx: u16, + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); +pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); +const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); + +allocator: std.mem.Allocator, +io: std.Io, +devices: []Device, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { + var monitor: Monitor = .{ + .allocator = allocator, + .io = io, + .devices = try discoverIntelDevices(allocator, io), + }; + try monitor.seedDeviceMetrics(); + return monitor; +} + +pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + const dev = try self.device(handle); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); + const raw = try smi_sysfs.readString(allocator, self.io, path); + return std.mem.trim(u8, raw, &std.ascii.whitespace); +} + +inline fn seedDeviceMetrics(self: *Monitor) !void { + var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; + defer device_usage.deinit(self.allocator); + + for (self.devices, 0..) |*dev, idx| { + dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; + if (device_usage.get(@intCast(idx))) |sample| { + dev.activity_prev = sample.engine; + dev.encoder_prev = sample.encoder; + dev.decoder_prev = sample.decoder; + } + } +} + +pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { + previous.clearRetainingCapacity(); + try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); + + var it = usage.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.engine) |engine| { + previous.putAssumeCapacity(entry.key_ptr.*, engine); + } + } +} + +inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { + var devices: std.ArrayList(Device) = .empty; + errdefer devices.deinit(allocator); + + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return &.{}, + else => |e| return e, + }; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + const render_idx = parseRenderIndex(entry.name) orelse continue; + const dev = openDevice(allocator, io, render_idx) catch continue; + try devices.append(allocator, dev); + } + + std.mem.sort(Device, devices.items, {}, struct { + fn lessThan(_: void, lhs: Device, rhs: Device) bool { + return lhs.render_idx < rhs.render_idx; + } + }.lessThan); + return devices.toOwnedSlice(allocator); +} + +inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { + const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); + errdefer allocator.free(drm_path); + + const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); + errdefer allocator.free(dev_path); + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); + const vendor = try smi_sysfs.readString(allocator, io, vendor_path); + if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { + return error.not_intel; + } + + const pcie_path = try findPcieLinkPath(allocator, io, dev_path); + errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); + + return .{ + .render_idx = render_idx, + .drm_path = drm_path, + .dev_path = dev_path, + .pcie_path = pcie_path, + .hwmon_path = findHwmon(allocator, io, dev_path) catch null, + .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, + }; +} + +pub fn device(self: *const Monitor, handle: Handle) !*const Device { + const idx = @backingInt(handle); + if (idx >= self.devices.len) return error.NotFound; + return &self.devices[idx]; +} + +pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch + readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch + readNumberedInput(allocator, io, hwmon, "temp", 2) catch + try readNumberedInput(allocator, io, hwmon, "temp", 1); + return (milli_c + 500) / 1000; +} + +pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch + readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch + try readNumberedInput(allocator, io, hwmon, "energy", 1); + return .{ + .micro_joules = micro_joules, + .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), + }; +} +pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch + try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); + return microwatts / 1000; +} + +pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); +} + +pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); +} + +pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "max_freq"); +} + +pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); + return readLargestResource(allocator, io, path); +} + +pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); + const raw = try smi_sysfs.readString(allocator, io, path); + return pcieGenFromSpeed(raw); +} + +pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); + return smi_sysfs.readInt(allocator, io, path); +} + +pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const gen = try readPcieLinkGen(allocator, io, dev); + const width = try readPcieLinkWidth(allocator, io, dev); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { + var processes = try collectProcessUsage(allocator, io, devices); + errdefer processes.deinit(allocator); + + var result: DeviceUsage = .{}; + errdefer result.deinit(allocator); + + var it = processes.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + const gop = try result.getOrPut(allocator, sample.device_idx); + if (!gop.found_existing) gop.value_ptr.* = .{}; + if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; + addEngineSample(&gop.value_ptr.engine, sample.engine); + addEngineSample(&gop.value_ptr.encoder, sample.encoder); + addEngineSample(&gop.value_ptr.decoder, sample.decoder); + } + + return result; +} + +pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { + var result: ProcessUsage = .{}; + errdefer result.deinit(allocator); + + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(allocator); + + var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); + defer proc_dir.close(io); + + const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; + var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; + defer fdinfo_dir.close(io); + + var fd_it = fdinfo_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; + const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; + const bus_device_identifier = parsed.bus_device_identifier orelse continue; + const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; + + // Skip fd if DRM client has been seen for this device. + // Summing after dup() / fork() over-reports memory and compute. + if (parsed.drm_client_id) |cid| { + if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; + } + + const key = processKey(pid, dev_idx); + const gop = try result.getOrPut(allocator, key); + if (!gop.found_existing) { + gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; + } + mergeProcessSample(gop.value_ptr, parsed.sample); + } + } + + return result; +} + +pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { + const prev = previous orelse return null; + const cur = current orelse return null; + if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; + const engine_delta = cur.engine_ns - prev.engine_ns; + const time_delta = cur.timestamp_ns - prev.timestamp_ns; + if (time_delta == 0) return null; + const pct = @min(engine_delta * 100 / time_delta, 100); + return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); +} + +pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { + const prev = previous orelse return null; + const cur = current orelse return null; + return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); +} + +inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { + if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; + const energy_delta = cur_uj - prev_uj; + const time_delta = cur_ns - prev_ns; + if (time_delta == 0) return null; + return energy_delta * 1_000_000 / time_delta; +} + +inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn parseRenderIndex(name: []const u8) ?usize { + if (!std.mem.startsWith(u8, name, "renderD")) return null; + return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; +} + +inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); + var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; + const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); + var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); + const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; + const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); + if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; + } + return null; +} + +inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); + const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (trimmed.len != bus_device_identifier_len) return error.NotFound; + var out: [bus_device_identifier_len]u8 = undefined; + @memcpy(&out, trimmed); + return out; +} + +inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { + var candidate = dev_path; + var best_path: ?[]const u8 = null; + + var best_bandwidth: u64 = 0; + var depth: usize = 0; + while (depth < 8) : (depth += 1) { + if (!try isIntelPciDevice(allocator, io, candidate)) break; + + const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; + if (bandwidth > best_bandwidth) { + best_path = candidate; + best_bandwidth = bandwidth; + } + + const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); + candidate = parent; + } + + return best_path orelse dev_path; +} + +inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); + const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; + + const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); + return std.mem.eql(u8, vendor, "0x8086"); +} + +inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { + var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); + const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); + + var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); + const width = try smi_sysfs.readInt(allocator, io, width_path); + const gen = try pcieGenFromSpeed(raw_speed); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { + for (1..16) |idx| { + var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); + const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; + return readNumberedInput(allocator, io, hwmon, prefix, idx); + } + return error.NotFound; +} + +inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); +} + +inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; + return pwmToPercent(pwm); + } + return error.NotFound; +} + +inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; + const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; + if (max == 0) continue; + return @min(input * 100 / max, 100); + } + return error.NotFound; +} + +inline fn pwmToPercent(pwm: u64) u64 { + return @min((@min(pwm, 255) * 100 + 127) / 255, 100); +} + +inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var line_writer: std.Io.Writer.Allocating = .init(allocator); + defer line_writer.deinit(); + + var largest: u64 = 0; + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + reader.interface.toss(1); + largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); + } + if (largest == 0) return error.NotFound; + return largest; +} + +inline fn resourceLineSize(line: []const u8) ?u64 { + var it = std.mem.tokenizeAny(u8, line, " \t"); + const base = parseHex(it.next() orelse return null) orelse return null; + const end = parseHex(it.next() orelse return null) orelse return null; + if (end < base) return null; + const size = end - base + 1; + return if (size >= 256 * 1024 * 1024) size else null; +} + +inline fn parseHex(raw: []const u8) ?u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; + return std.fmt.parseInt(u64, digits, 16) catch null; +} + +inline fn pcieGenFromSpeed(raw: []const u8) !u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; + if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; + if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; + if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; + if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; + if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; + return error.NotFound; +} + +inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { + const lane_mb_s: u64 = switch (gen) { + 1 => 250, + 2 => 500, + 3 => 985, + 4 => 1969, + 5 => 3938, + 6 => 7563, + else => return null, + }; + return lane_mb_s * width; +} + +const FdinfoSample = struct { + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +const ParsedFdinfo = struct { + bus_device_identifier: ?[bus_device_identifier_len]u8 = null, + drm_client_id: ?u64 = null, + sample: FdinfoSample = .{}, + resident_vram_kib: u64 = 0, + total_vram_kib: u64 = 0, + resident_memory_kib: u64 = 0, + total_memory_kib: u64 = 0, + cycle_counter: u64 = 0, + total_cycle_counter: u64 = 0, + engine_time_seen: bool = false, +}; + +inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { + var read_buf: [8192]u8 = undefined; + const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; + + var parsed: ParsedFdinfo = .{}; + + var lines = std.mem.tokenizeScalar(u8, data, '\n'); + while (lines.next()) |line| { + parseFdinfoLine(&parsed, line, timestamp_ns); + } + + finishFdinfo(&parsed); + return parsed; +} + +inline fn finishFdinfo(self: *ParsedFdinfo) void { + if (self.resident_vram_kib > 0) { + self.sample.mem_kib = self.resident_vram_kib; + } else if (self.total_vram_kib > 0) { + self.sample.mem_kib = self.total_vram_kib; + } else if (self.resident_memory_kib > 0) { + self.sample.mem_kib = self.resident_memory_kib; + } else if (self.total_memory_kib > 0) { + self.sample.mem_kib = self.total_memory_kib; + } + + if (!self.engine_time_seen and self.total_cycle_counter > 0) { + self.sample.engine = .{ + .engine_ns = self.cycle_counter, + .timestamp_ns = self.total_cycle_counter, + }; + } +} + +inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { + if (std.mem.startsWith(u8, line, "drm-pdev:")) { + parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-client-id:")) { + parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "vram mem:")) { + parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.cycle_counter += firstInt(line[colon + 1 ..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const engine_label = line["drm-engine-".len..colon]; + const value = firstInt(line[colon + 1 ..]); + parsed.engine_time_seen = true; + addEngineTime(&parsed.sample.engine, value, timestamp_ns); + if (containsAsciiIgnoreCase(engine_label, "encode")) { + addEngineTime(&parsed.sample.encoder, value, timestamp_ns); + } else if (containsAsciiIgnoreCase(engine_label, "decode")) { + addEngineTime(&parsed.sample.decoder, value, timestamp_ns); + } + return; + } + + if (std.mem.startsWith(u8, line, "drm-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const key = line[0..colon]; + const value = memoryKiB(line[colon + 1 ..]); + if (value == 0) return; + + if (std.mem.startsWith(u8, key, "drm-resident-vram")) { + parsed.resident_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { + parsed.total_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-resident-")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-")) { + parsed.total_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-resident-memory")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-total-memory")) { + parsed.total_memory_kib += value; + return; + } + } +} + +fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { + if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; + addEngineSample(&dst.engine, src.engine); + addEngineSample(&dst.encoder, src.encoder); + addEngineSample(&dst.decoder, src.decoder); +} + +fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { + if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); +} + +fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { + const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; + dst.* = .{ + .engine_ns = current.engine_ns + engine_ns, + .timestamp_ns = timestamp_ns, + }; +} + +fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { + for (devices, 0..) |dev, idx| { + if (dev.bus_device_identifier) |device_bus_device_identifier| { + if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); + } + } + return null; +} + +fn processKey(pid: u32, device_idx: u16) u64 { + return (@as(u64, device_idx) << 32) | pid; +} + +pub const BdfParts = struct { + domain: u32, + bus: u32, + device: u32, + function: u32, +}; + +pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { + var buf: [bus_device_identifier_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + parts.domain, + parts.bus, + parts.device, + parts.function, + }) catch unreachable; + return buf; +} + +inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { + const trimmed = std.mem.trim(u8, raw, " \t\n"); + var out: [bus_device_identifier_len]u8 = undefined; + if (trimmed.len == bus_device_identifier_len) { + @memcpy(&out, trimmed[0..bus_device_identifier_len]); + return out; + } + if (trimmed.len == "00:00.0".len) { + _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; + return out; + } + return null; +} + +inline fn firstInt(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; +} + +inline fn memoryKiB(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; + const unit = iter.next() orelse return value; + if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; + if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; + if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; + if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; + return value; +} + +inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |i| { + if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; + } + return false; +} + +test "oneAPI fdinfo parser" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); + parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); + parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expect(parsed.bus_device_identifier != null); + try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); + try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); + try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser classifies explicit media engines" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); + try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); +} + +test "oneAPI fdinfo parser leaves generic video engines unclassified" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser cycle counters" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); + parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); + parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); +} + +test "oneAPI fdinfo parser xe vram regions" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); + parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); + parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); +} + +test "oneAPI fdinfo parser falls back to total vram" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); +} + +test "oneAPI DRM clients dedup per device" { + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(std.testing.allocator); + + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); +} + +test "oneAPI process utilization delta" { + try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); +} + +test "oneAPI PWM fan speed conversion" { + try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); + try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); +} + +test "oneAPI process previous keeps engine samples" { + var usage: ProcessUsage = .{}; + defer usage.deinit(std.testing.allocator); + + const engine_key = processKey(10, 1); + const mem_only_key = processKey(20, 1); + try usage.put(std.testing.allocator, engine_key, .{ + .pid = 10, + .device_idx = 1, + .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, + }); + try usage.put(std.testing.allocator, mem_only_key, .{ + .pid = 20, + .device_idx = 1, + .mem_kib = 2048, + }); + + var previous: ProcessPrevious = .{}; + defer previous.deinit(std.testing.allocator); + + try saveProcessPrevious(std.testing.allocator, &previous, &usage); + + try std.testing.expectEqual(@as(u32, 1), previous.count()); + try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); + try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); +} + +test "oneAPI power delta from hwmon energy" { + try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); +} + +test "oneAPI resource line chooses large memory BARs" { + try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); + try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); +} + +test "oneAPI PCIe speed maps to generation" { + try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); + try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); + try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); +} + +test "oneAPI PCIe bandwidth derives from generation and width" { + try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); + try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); + try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); +} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..2e1e5deb --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -0,0 +1,248 @@ +const std = @import("std"); +const Monitor = @import("monitor.zig"); +const smi_sysfs = @import("zml-smi/sysfs"); + +const OneApi = @This(); + +pub const Handle = Monitor.Handle; + +monitor: Monitor, +io: std.Io, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { + return .{ + .monitor = try Monitor.init(allocator, io), + .io = io, + }; +} + +pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { + if (device_id >= self.monitor.devices.len) return error.NotFound; + return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); +} + +pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + if (self.monitor.device(handle)) |dev| { + if (dev.bus_device_identifier) |bdf| { + if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { + return display_name; + } + } + } else |_| {} + + const id = try self.monitor.deviceId(allocator, handle); + return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); +} + +fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { + const result = std.process.run(allocator, io, .{ + .argv = &.{ "lspci", "-Dnn", "-s", bdf }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(1024), + .reserve_amount = 256, + }) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return null, + else => return null, + } + + return parseLspciName(allocator, result.stdout); +} + +fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { + var lines = std.mem.tokenizeScalar(u8, output, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); + const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; + const header = trimmed[0 .. header_end + 1]; + if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; + + const name_start = header_end + "]: ".len; + const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse + std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse + trimmed[name_start..].len; + const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); + if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; + return try allocator.dupe(u8, raw_name); + } + return null; +} + +fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |idx| { + if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; + } + return false; +} + +test "parse lspci display name" { + const output = + \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) + ; + const name_ = (try parseLspciName(std.testing.allocator, output)).?; + defer std.testing.allocator.free(name_); + try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); +} + +test "parse lspci display name skips generic device ids" { + const output = + \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) + ; + try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); +} + +pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { + const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch + try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); + const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); + if (trimmed.len == 0) return error.NotFound; + return trimmed; +} + +pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { + dev.energy_prev = null; + return err; + }; + const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { + dev.energy_prev = current; + return error.NotFound; + }; + dev.energy_prev = current; + return power; +} + +pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPowerLimit(allocator, self.io, dev); +} + +pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readTemperature(allocator, self.io, dev); +} + +pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readFanSpeed(allocator, self.io, dev); +} + +pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { + dev.activity_prev = null; + return 0; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.engine else null; + const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; + dev.activity_prev = current; + return util; +} + +pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.encoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; + const util = Monitor.processUtil(dev.encoder_prev, current) orelse { + dev.encoder_prev = current; + return error.NotFound; + }; + dev.encoder_prev = current; + return util; +} + +pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.decoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; + const util = Monitor.processUtil(dev.decoder_prev, current) orelse { + dev.decoder_prev = current; + return error.NotFound; + }; + dev.decoder_prev = current; + return util; +} + +pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readClockGraphics(allocator, self.io, dev); +} + +pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMaxClockGraphics(allocator, self.io, dev); +} + +pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev_idx: u16 = @intCast(@backingInt(handle)); + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; + defer usage.deinit(allocator); + + return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; +} + +pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMemTotal(allocator, self.io, dev); +} + +pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkGen(allocator, self.io, dev); +} + +pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkWidth(allocator, self.io, dev); +} + +pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieBandwidth(allocator, self.io, dev); +} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig new file mode 100644 index 00000000..c4d0c719 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/process.zig @@ -0,0 +1,45 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const Monitor = @import("monitor.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { + const state = try collector.arena.create(Monitor.ProcessPrevious); + state.* = .{}; + + const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; + if (previous_usage) |usage_value| { + var usage = usage_value; + defer usage.deinit(collector.gpa); + Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); +} + +fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { + const back = list.back(); + back.clearRetainingCapacity(); + + var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { + list.swap(); + return; + }; + defer usage.deinit(allocator); + + var it = usage.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + back.append(allocator, .{ + .pid = sample.pid, + .device_idx = @intCast(sample.device_idx + dev_offset), + .dev_mem_kib = sample.mem_kib, + .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), + }) catch break; + } + + Monitor.saveProcessPrevious(arena, state, &usage) catch {}; + list.swap(); +} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel new file mode 100644 index 00000000..a65377b3 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "tpuinfo", + srcs = ["tpuinfo.cc"], + hdrs = ["tpuinfo.h"], + deps = [ + "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", + "@grpc//:grpc++_unsecure", + ], +) + +zig_library( + name = "tpu", + import_name = "zml-smi/platforms/tpu", + main = "metrics.zig", + srcs = [ + "process.zig", + "tpuinfo.zig", + ], + visibility = ["//visibility:public"], + deps = [ + ":tpuinfo", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig new file mode 100644 index 00000000..9410b3f8 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -0,0 +1,168 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const tpuinfo = @import("tpuinfo.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const TpuInfo = device_info.TpuInfo; +const Collector = @import("zml-smi/collector").Collector; +const tpu_process = @import("process.zig"); + +const address = "localhost:8431"; + +pub fn start(collector: *Collector) !void { + const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; + const device_count = chip.chip_count * chip.devices_per_chip; + var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..device_count) |_| { + const chip_name = try collector.arena.dupe(u8, chip.name); + const initial: TpuInfo = .{ .name = chip_name }; + const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); + try tpu_infos.append(collector.arena, info); + } + + try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); + + if (tpu_infos.items.len > 0) { + const processes = try collector.createProcessList(); + try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); + } +} + +const MetricKind = enum { int, double }; + +fn pollAllDevices(devs: []*DeviceInfo) void { + var mem_used: [tpuinfo.max_devices]?u64 = undefined; + var mem_total: [tpuinfo.max_devices]?u64 = undefined; + var util: [tpuinfo.max_devices]?u64 = undefined; + + for (devs, 0..) |info, i| { + const front = info.tpu.front(); + mem_used[i] = front.mem_used_bytes; + mem_total[i] = front.mem_total_bytes; + util[i] = front.util_percent; + } + + queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); + queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); + + for (devs, 0..) |info, i| { + const back = info.tpu.back(); + back.* = info.tpu.front().*; + back.mem_used_bytes = mem_used[i]; + back.mem_total_bytes = mem_total[i]; + back.util_percent = util[i]; + info.tpu.swap(); + } +} + +fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { + var ids: [tpuinfo.max_devices]c_longlong = undefined; + switch (kind) { + .int => { + var vals: [tpuinfo.max_devices]c_longlong = undefined; + const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); + } + }, + .double => { + var vals: [tpuinfo.max_devices]f64 = undefined; + const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len and std.math.isFinite(val)) + out[@intCast(id)] = @intFromFloat(@round(val)); + } + }, + } +} + +// -- Discover available chips +// We need to scan /sys/bus/pci/devices for Google TPU chips. +// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 + +const ChipInfo = struct { + name: []const u8, + devices_per_chip: u32, + chip_count: u32, +}; + +const google_pci_vendor_id = "0x1ae0"; +const pci_base = "/sys/bus/pci/devices"; + +const ChipEntry = struct { + device_id: []const u8, + subsystem_id: ?[]const u8 = null, + name: []const u8, + devices_per_chip: u32, +}; + +const chip_table = [_]ChipEntry{ + .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, + .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, + .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, + .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, + .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, + .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, + .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, +}; + +fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { + var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; + defer pci_dir.close(io); + + var it = pci_dir.iterate(); + var chip_count: u32 = 0; + var result: ChipInfo = undefined; + + while (it.next(io) catch null) |entry| { + const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; + if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { + continue; + } + + const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; + const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; + + for (chip_table) |chip| { + const dev_match = std.mem.eql(u8, device_raw, chip.device_id); + const sub_match = if (chip.subsystem_id) |expected_sub| + if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false + else + true; + + if (dev_match and sub_match) { + result = .{ + .name = chip.name, + .devices_per_chip = chip.devices_per_chip, + .chip_count = undefined, + }; + + chip_count += 1; + break; + } + } + } + + if (chip_count > 0) { + result.chip_count = chip_count; + return result; + } + + return null; +} + +fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { + const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); + return sysfs.readString(allocator, io, path); +} diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig new file mode 100644 index 00000000..cca8f6a1 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/process.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { + scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); + list.swap(); +} + +fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { + back.clearRetainingCapacity(); + + var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; + defer proc_dir.close(io); + + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; + var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; + defer fd_dir.close(io); + + var fd_it = fd_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; + const target = link_buf[0..len]; + + if (parseChipIndex(target)) |chip_idx| { + const base: u16 = @intCast(chip_idx * devices_per_chip); + for (0..devices_per_chip) |d| { + const local_idx: u16 = @intCast(base + d); + var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; + + if (local_idx < infos.len) { + const tpu = infos[local_idx].tpu.front().*; + + if (tpu.mem_used_bytes) |mem| { + info.dev_mem_kib = @intCast(mem / 1024); + } + + if (tpu.util_percent) |util| { + info.dev_util_percent = @intCast(util); + } + } + + back.append(allocator, info) catch return; + } + } + } + } +} + +fn parseChipIndex(target: []const u8) ?u32 { + if (std.mem.startsWith(u8, target, "/dev/accel")) { + return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; + } + + if (std.mem.startsWith(u8, target, "/dev/vfio/")) { + return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; + } + + return null; +} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.cc b/bin/zml-smi/platforms/tpu/tpuinfo.cc new file mode 100644 index 00000000..a1e3fbda --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.cc @@ -0,0 +1,84 @@ +#include "tpuinfo.h" + +#include +#include +#include +#include + +#include + +#include "tpu_metric_service.grpc.pb.h" +#include "tpu_metric_service.pb.h" + +using tpu::monitoring::runtime::MetricRequest; +using tpu::monitoring::runtime::MetricResponse; +using tpu::monitoring::runtime::RuntimeMetricService; + +static MetricResponse FetchMetric(const char* address, const char* metric_name, + bool* ok) { + auto channel = + grpc::CreateChannel(address, grpc::InsecureChannelCredentials()); + auto stub = RuntimeMetricService::NewStub(channel); + + grpc::ClientContext ctx; + ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(1)); + + MetricRequest req; + req.set_metric_name(metric_name); + + MetricResponse resp; + auto status = stub->GetRuntimeMetric(&ctx, req, &resp); + *ok = status.ok(); + + return resp; +} + +extern "C" int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_int(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} + +extern "C" int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_double(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.h b/bin/zml-smi/platforms/tpu/tpuinfo.h new file mode 100644 index 00000000..f2dc7db6 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.h @@ -0,0 +1,20 @@ +#ifndef TPUINFO_H_ +#define TPUINFO_H_ + +#define TPUINFO_MAX_DEVICES 64 + +#ifdef __cplusplus +extern "C" { +#endif + +int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, int max_n); + +int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, int max_n); + +#ifdef __cplusplus +} +#endif + +#endif /* TPUINFO_H_ */ diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig new file mode 100644 index 00000000..1baf62df --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const c = @import("c"); + +pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; + +pub const Error = error{ + TpuUnavailable, + QueryFailed, +}; + +pub fn queryInt( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []c_longlong, +) Error!u32 { + const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} + +pub fn queryDouble( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []f64, +) Error!u32 { + const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel new file mode 100644 index 00000000..c00a26b5 --- /dev/null +++ b/bin/zml-smi/prometheus/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "prometheus", + import_name = "zml-smi/prometheus", + main = "prometheus.zig", + srcs = [ + "server.zig", + "exposition.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig new file mode 100644 index 00000000..54d48cb6 --- /dev/null +++ b/bin/zml-smi/prometheus/exposition.zig @@ -0,0 +1,159 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; + +const device_metrics = .{ + // Shared across all device types + .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, + .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, + .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, + // GPU (CUDA/ROCm/oneAPI) + .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, + .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, + .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, + .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, + .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, + .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, + .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, + .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, + .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, + // Neuron + .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, + .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, + .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, + .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, + .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, + .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, + .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, + .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, + .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, + .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, + .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, +}; + +const host_metrics = .{ + .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, + .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, + .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, + .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, + .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, +}; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { + try writer.writeAll("# HELP zml_device_info Device metadata\n"); + try writer.writeAll("# TYPE zml_device_info gauge\n"); + for (devices, 0..) |dev, i| { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { + const tag = @field(smi_info.device_info.Target, type_name); + + switch (dev.*) { + tag => |*db| { + const val = db.front().*; + + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); + + inline for (.{ + .{ "name", "name" }, + .{ "driver_version", "driver" }, + .{ "cuda_driver_version", "cuda_version" }, + }) |label| { + if (@hasField(@TypeOf(val), label[0])) { + if (@field(val, label[0])) |v| { + try writer.print(",{s}=\"", .{label[1]}); + try std.zig.stringEscape(v, writer); + try writer.writeAll("\""); + } + } + } + + try writer.writeAll("} 1\n"); + }, + else => {}, + } + } + } + try writer.writeAll("\n"); + + try writeAllDeviceMetrics(writer, device_metrics, devices); + try writeHostMetrics(writer, host.front().*); +} + +fn writeAllDeviceMetrics( + writer: *std.Io.Writer, + comptime metrics: anytype, + devices: []const *DeviceInfo, +) !void { + inline for (metrics) |m| { + var header_written = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + inline else => |*db| { + const val = db.front().*; + if (@hasField(@TypeOf(val), m.field)) { + if (@field(val, m.field)) |raw| { + if (!header_written) { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); + header_written = true; + } + + const device_type = @tagName(std.meta.activeTag(dev.*)); + try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); + try std.zig.stringEscape(val.name orelse "", writer); + try writer.writeAll("\"} "); + try fmtValue(m, writer, raw); + try writer.writeAll("\n"); + } + } + }, + } + } + + if (header_written) { + try writer.writeAll("\n"); + } + } +} + +fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { + inline for (host_metrics) |m| { + if (@field(val, m.field)) |raw| { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); + try fmtValue(m, writer, raw); + try writer.writeAll("\n\n"); + } + } +} + +fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { + if (@hasField(@TypeOf(m), "fmt")) { + try m.fmt(writer, raw); + } else { + try writeValue(writer, raw); + } +} + +fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float, .comptime_float => try writer.print("{d:.2}", .{val}), + .pointer => try writer.writeAll(val), + else => try writer.print("{d}", .{val}), + } +} + +fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); +} + +fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { + return struct { + fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}", .{val * scale}); + } + }.f; +} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig new file mode 100644 index 00000000..cfa2fc28 --- /dev/null +++ b/bin/zml-smi/prometheus/prometheus.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig new file mode 100644 index 00000000..9517ecac --- /dev/null +++ b/bin/zml-smi/prometheus/server.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; +const exposition = @import("exposition.zig"); + +pub const Server = struct { + io: std.Io, + devices: []const *DeviceInfo, + host: *const HostInfo, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { + const address = try std.Io.net.IpAddress.parseLiteral(listen); + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .io = io, + .devices = devices, + .host = host, + .tcp = tcp, + }; + } + + pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { + var self = init(io, listen, devices, host_info) catch |err| { + std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + if (request.head.method != .GET) { + try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); + continue; + } + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "text/plain" }, + }, + }, + }); + + try exposition.write(&body_writer.writer, self.devices, self.host); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh new file mode 100755 index 00000000..d1e89902 --- /dev/null +++ b/bin/zml-smi/scripts/release.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- UI --- + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*"; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +confirm() { + if [ "$YES" = true ]; then + return 0 + fi + printf "${WHITE} ? %b ${DIM}[y/N]${RESET} " "$*" + local answer + read -r answer + case "$answer" in + y|Y|yes|Yes) return 0 ;; + *) return 1 ;; + esac +} + +usage() { + printf >&2 " + ${WHITE}Usage${RESET} + bazel run //bin/zml-smi:release -- [flags] + + ${WHITE}Arguments${RESET} + ${CYAN}version${RESET} Version string, e.g. ${DIM}v1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) + + ${WHITE}Flags${RESET} + ${CYAN}--dry-run${RESET} Build only — skip tagging, uploading, and publishing + ${CYAN}--verify${RESET} Verify uploaded files by comparing sha256 against the CDN + ${CYAN}--publish${RESET} Push git tag and create a GitHub release + ${CYAN}-y, --yes${RESET} Skip all confirmation prompts + ${CYAN}-h, --help${RESET} Show this help + + ${WHITE}Environment${RESET} + ${CYAN}BASE_URL${RESET} Public base URL serving the bucket + ${CYAN}RCLONE_REMOTE${RESET} Rclone remote name ${DIM}(default: r2)${RESET} + + ${DIM}If rclone is not configured, you can set the remote via env vars:${RESET} + ${DIM} RCLONE_CONFIG_R2_TYPE=s3${RESET} + ${DIM} RCLONE_CONFIG_R2_PROVIDER=Cloudflare${RESET} + ${DIM} RCLONE_CONFIG_R2_ACCESS_KEY_ID=your_access_key${RESET} + ${DIM} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=your_secret_key${RESET} + ${DIM} RCLONE_CONFIG_R2_ENDPOINT=https://.r2.cloudflarestorage.com${RESET} + + ${WHITE}Examples${RESET} + ${DIM}# Interactive, local only${RESET} + bazel run //bin/zml-smi:release -- 0.4.0 + + ${DIM}# Full release, non-interactive${RESET} + bazel run //bin/zml-smi:release -- --publish --verify --yes 0.4.0 + +" + exit 0 +} + +# --- Checks --- + +check_env() { + if [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]; then + fail "Must be run via 'bazel run //bin/zml-smi:release -- '" + fi + if [ -z "${BASE_URL:-}" ]; then + fail "Missing required environment variable: ${CYAN}BASE_URL" + fi +} + +# --- Steps --- + +changelog() { + local prev_tag + prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-*' 2>/dev/null || true)" + local range="${prev_tag:+${prev_tag}..HEAD}" + + git log ${range:-HEAD} --pretty=format:"- %s" -- bin/zml-smi/ | grep -vi WIP || true +} + +tag() { + step "Tagging ${CYAN}${TAG}${WHITE}" + + # Cache changelog before tagging, otherwise the new tag makes the range empty + CACHED_CHANGELOG="$(changelog)" + + if [ -z "$CACHED_CHANGELOG" ]; then + warn "No changes since last release" + fi + + if git rev-parse "$TAG" &>/dev/null; then + warn "Tag ${TAG} already exists, skipping" + return + fi + + if [ "$DRY_RUN" = true ]; then + warn "Dry run — skipping tag creation" + elif confirm "Create tag ${CYAN}${TAG}${RESET}${WHITE}?"; then + git tag -a "$TAG" -m "$(printf '%s\n\n%s' "$TAG" "$CACHED_CHANGELOG")" + success "Created tag ${CYAN}${TAG}" + else + warn "Skipped tagging" + return + fi + printf "\n" + + printf "${DIM}" + echo "$CACHED_CHANGELOG" + printf "${RESET}" +} + +build() { + step "Building release archives" + + if ! confirm "Build all platforms?"; then + warn "Skipped build" + return 1 + fi + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + + bazel build --config=release //bin/zml-smi:archives + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + FILES=() + while IFS= read -r line; do + FILES+=("$line") + done < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) + success "Built ${CYAN}${#FILES[@]}${RESET}${GREEN} artifacts" +} + +upload() { + local rclone_remote="${RCLONE_REMOTE:-r2}" + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Patching install script" + + INSTALL_SCRIPT="$(mktemp)" + sed -e "s|^BASE_URL=.*|BASE_URL=\"${install_url}\"|" \ + -e "s|^VERSION=.*|VERSION=\"${VERSION}\"|" \ + "${BUILD_WORKSPACE_DIRECTORY}/bin/zml-smi/scripts/remote-install.sh" > "$INSTALL_SCRIPT" + success "BASE_URL=${CYAN}${install_url}" + + if [ "$DRY_RUN" = true ]; then + step "Uploading to ${CYAN}${rclone_remote}" + warn "Dry run — skipping upload" + return + fi + + step "Uploading to ${CYAN}${rclone_remote}" + + if ! confirm "Upload artifacts?"; then + warn "Skipped upload" + return + fi + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + warn "SKIP $(basename "$f") ${DIM}(not built for this host)${RESET}" + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + rclone copyto "$f" "${rclone_remote}:zml-smi/${VERSION}/${remote_name}" + success "${remote_name}" + done + + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/${VERSION}/install.sh" + success "zml-smi/${VERSION}/install.sh" + + if confirm "Set ${CYAN}${VERSION}${RESET}${WHITE} as the default install version?"; then + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/install.sh" + success "zml-smi/install.sh" + DEFAULT_INSTALL=true + else + warn "Skipped default install script" + DEFAULT_INSTALL=false + fi +} + +verify() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Verifying uploads" + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + local local_sha remote_sha + local_sha="$(shasum -a 256 "$f" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/${remote_name}" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "${remote_name}" + else + fail "${remote_name} — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi + done + + local local_sha remote_sha + local_sha="$(shasum -a 256 "$INSTALL_SCRIPT" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/install.sh" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "install.sh" + else + fail "install.sh — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi +} + +release_notes() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + local log="$CACHED_CHANGELOG" + + local downloads="" + for f in "${FILES[@]}"; do + [ -f "$f" ] || continue + local name remote_name + name="$(basename "$f")" + [[ "$name" == *.sha256 ]] && continue + local sha_file="" + for sf in "${FILES[@]}"; do + if [ "$(basename "$sf")" = "${name}.sha256" ]; then + sha_file="$sf" + break + fi + done + if [ -z "$sha_file" ] || [ ! -f "$sha_file" ]; then + fail "Missing sha256 sidecar for ${name}" + fi + local sha + sha="$(awk '{print $1}' "$sha_file")" + local actual + actual="$(shasum -a 256 "$f" | awk '{print $1}')" + if [ "$sha" != "$actual" ]; then + fail "${name} sha256 mismatch — sidecar: ${sha}, actual: ${actual}" + fi + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + downloads="${downloads}| [${remote_name}](${install_url}/${remote_name}) | \`${sha}\` |"$'\n' + done + + cat </dev/null && [ -w "$INSTALL_DIR" ] && + mkdir -p "$BIN_DIR" 2>/dev/null && [ -w "$BIN_DIR" ]; then + return + fi + + warn "Cannot write to /opt or /usr/local/bin — falling back to ~/.local" + INSTALL_DIR="${HOME}/.local/zml-smi" + BIN_DIR="${HOME}/.local/bin" +} + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*" >&2; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +print_logo() { + printf "\n" + printf "${BLUE} █████╗███╗ ███╗██╗ ${RESET}\n" + printf "${BLUE} ╚═██╔╝████╗ ████║██║ ${RESET}\n" + printf "${BLUE} ██╔╝ ██╔████╔██║██║ ${RESET}\n" + printf "${BLUE} █████╗██║ ╚═╝ ██║████╗${RESET}\n" + printf "${BLUE} ╚════╝╚═╝ ╚═╝╚═══╝${RESET}\n" +} + +detect_platform() { + step "Detecting platform" + + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux) OS_LABEL="linux" ;; + Darwin) OS_LABEL="macos" ;; + MINGW*|MSYS*|CYGWIN*) + fail "Windows is not supported. Use WSL2 instead." ;; + *) + fail "Unsupported operating system: ${OS}" ;; + esac + + case "$ARCH" in + x86_64|amd64) ARCH_LABEL="amd64" ;; + aarch64|arm64) ARCH_LABEL="arm64" ;; + *) + fail "Unsupported architecture: ${ARCH}" ;; + esac + + if [ "$OS_LABEL" = "macos" ] && [ "$ARCH_LABEL" = "amd64" ]; then + fail "macOS x86_64 is not supported. Only Apple Silicon (arm64) is supported." + fi + + success "${OS_LABEL} ${ARCH_LABEL}" + + DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" +} + +check_deps() { + step "Checking dependencies" + local missing=() + + if command -v curl &>/dev/null; then + DOWNLOADER="curl" + success "curl" + elif command -v wget &>/dev/null; then + DOWNLOADER="wget" + success "wget" + else + missing+=("curl or wget") + fi + + if command -v zstd &>/dev/null; then + success "zstd" + else + missing+=("zstd") + fi + + if command -v tar &>/dev/null; then + success "tar" + else + missing+=("tar") + fi + + if [ ${#missing[@]} -gt 0 ]; then + fail "Missing required tools: ${missing[*]}" + fi +} + +download_and_install() { + step "Downloading & extracting" + + rm -rf "$INSTALL_DIR" + mkdir -p "$INSTALL_DIR" + + if [ "$DOWNLOADER" = "curl" ]; then + if ! curl -fsSL "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + else + if ! wget -q -O- "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + fi + + chmod 755 "${INSTALL_DIR}/${BINARY_NAME}" + success "Installed to ${INSTALL_DIR}/" + + mkdir -p "$BIN_DIR" + ln -sf "${INSTALL_DIR}/${BINARY_NAME}" "${BIN_DIR}/${BINARY_NAME}" + success "Symlink at ${BIN_DIR}/${BINARY_NAME}" +} + +check_path() { + if ! echo "$PATH" | tr ':' '\n' | grep -qx "$BIN_DIR"; then + printf "\n" + warn "${BIN_DIR} is not in your PATH" + step "Add to your shell profile:" + info "${CYAN}export PATH=\"${BIN_DIR}:\$PATH\"" + printf "\n" + fi +} + +main() { + print_logo + resolve_dirs + detect_platform + check_deps + + info "Install to: ${CYAN}${INSTALL_DIR}${RESET}" + info "Bin dir: ${CYAN}${BIN_DIR}${RESET}" + + download_and_install + check_path + + printf "\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf " ${GREEN} Installation complete!${RESET}\n" + printf " ${DIM} Run '${BINARY_NAME}' to get started.${RESET}\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf "\n" +} + +main diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel new file mode 100644 index 00000000..29f4824c --- /dev/null +++ b/bin/zml-smi/tui/BUILD.bazel @@ -0,0 +1,59 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "tui", + import_name = "zml-smi/tui", + main = "tui.zig", + srcs = [ + "data.zig", + "image_cache.zig", + "lib/compose.zig", + "lib/image.zig", + "lib/ui.zig", + "lib/utils.zig", + "pages/detail.zig", + "pages/detail/detail_common.zig", + "pages/detail/gpu_detail.zig", + "pages/detail/neuron_detail.zig", + "pages/detail/tpu_detail.zig", + "pages/overview.zig", + "print.zig", + "theme.zig", + "top.zig", + "widgets/braille_chart.zig", + "widgets/chart.zig", + "widgets/column_layout.zig", + "widgets/device_card.zig", + "widgets/gauge.zig", + "widgets/info_lines.zig", + "widgets/logo.zig", + "widgets/metric_card.zig", + "widgets/process_table.zig", + "widgets/status_line.zig", + "widgets/titled_border.zig", + ], + extra_srcs = [ + "assets/amd.png", + "assets/intel.png", + "assets/logo.png", + "assets/neuron.png", + "assets/nvidia.png", + "assets/tpu.png", + ], + visibility = ["//visibility:public"], + deps = [ + "//bazel/stamp:zig", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//zml:logo", + "@libvaxis//:vaxis", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) + +zig_test( + name = "test", + deps = [":tui"], +) diff --git a/bin/zml-smi/tui/assets/amd.png b/bin/zml-smi/tui/assets/amd.png new file mode 100644 index 00000000..1de65ece Binary files /dev/null and b/bin/zml-smi/tui/assets/amd.png differ diff --git a/bin/zml-smi/tui/assets/intel.png b/bin/zml-smi/tui/assets/intel.png new file mode 100644 index 00000000..1f71fcd7 Binary files /dev/null and b/bin/zml-smi/tui/assets/intel.png differ diff --git a/bin/zml-smi/tui/assets/logo.png b/bin/zml-smi/tui/assets/logo.png new file mode 100644 index 00000000..77ceba76 Binary files /dev/null and b/bin/zml-smi/tui/assets/logo.png differ diff --git a/bin/zml-smi/tui/assets/neuron.png b/bin/zml-smi/tui/assets/neuron.png new file mode 100644 index 00000000..ea57a753 Binary files /dev/null and b/bin/zml-smi/tui/assets/neuron.png differ diff --git a/bin/zml-smi/tui/assets/nvidia.png b/bin/zml-smi/tui/assets/nvidia.png new file mode 100644 index 00000000..6c2a474c Binary files /dev/null and b/bin/zml-smi/tui/assets/nvidia.png differ diff --git a/bin/zml-smi/tui/assets/tpu.png b/bin/zml-smi/tui/assets/tpu.png new file mode 100644 index 00000000..97a6ae3c Binary files /dev/null and b/bin/zml-smi/tui/assets/tpu.png differ diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig new file mode 100644 index 00000000..bc6171b7 --- /dev/null +++ b/bin/zml-smi/tui/data.zig @@ -0,0 +1,155 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const di = smi_info.device_info; +pub const DeviceInfo = di.DeviceInfo; +pub const GpuInfo = di.GpuInfo; +pub const NeuronInfo = di.NeuronInfo; +pub const TpuInfo = di.TpuInfo; +pub const Target = di.Target; +pub const Targets = smi_info.Targets; +const hi = smi_info.host_info; +pub const HostInfo = hi.HostInfo; +pub const HostData = hi.HostData; +pub const pi = smi_info.process_info; +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +pub const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const history_len: usize = 500; + +pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { + return struct { + const Self = @This(); + + buf: [capacity]T = @splat(0), + head: usize = 0, + len: usize = 0, + + pub fn push(self: *Self, value: T) void { + self.buf[self.head] = value; + self.head = (self.head + 1) % capacity; + if (self.len < capacity) { + self.len += 1; + } + } + + pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { + const count = @min(n, self.len); + if (count == 0) { + return &.{}; + } + const out = try arena.alloc(T, count); + const start = (self.head + capacity - count) % capacity; + for (0..count) |i| { + out[i] = self.buf[(start + i) % capacity]; + } + return out; + } + }; +} + +pub const HistoryBuffers = struct { + util: []RingBuffer(u64, history_len) = &.{}, + mem_util: []RingBuffer(u64, history_len) = &.{}, + temp: []RingBuffer(u64, history_len) = &.{}, + power: []RingBuffer(u64, history_len) = &.{}, + + pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { + const h: HistoryBuffers = .{ + .util = try allocator.alloc(RingBuffer(u64, history_len), count), + .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), + .temp = try allocator.alloc(RingBuffer(u64, history_len), count), + .power = try allocator.alloc(RingBuffer(u64, history_len), count), + }; + + @memset(h.util, .{}); + @memset(h.mem_util, .{}); + @memset(h.temp, .{}); + @memset(h.power, .{}); + + return h; + } + + pub fn deinit(self: *HistoryBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.util); + allocator.free(self.mem_util); + allocator.free(self.temp); + allocator.free(self.power); + } +}; + +pub const SystemState = struct { + devices: []*DeviceInfo, + host: *HostInfo, + history: HistoryBuffers = .{}, + targets: Targets, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, + io: std.Io, + tui_refresh_rate: u16, + + pub const Config = struct { + devices: []*DeviceInfo, + host: *HostInfo, + targets: Targets, + tui_refresh_rate: u16, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, + io: std.Io, + }; + + pub fn init(cfg: Config) !SystemState { + return .{ + .devices = cfg.devices, + .host = cfg.host, + .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), + .targets = cfg.targets, + .tui_refresh_rate = cfg.tui_refresh_rate, + .process_lists = cfg.process_lists, + .enricher = cfg.enricher, + .gpa = cfg.gpa, + .arena = cfg.arena, + .io = cfg.io, + }; + } + + pub fn deinit(self: *SystemState, allocator: std.mem.Allocator) void { + self.history.deinit(allocator); + } + + pub fn deviceCount(self: *const SystemState) usize { + return self.devices.len; + } + + pub fn recordHistory(self: *SystemState) void { + for (0..self.deviceCount()) |i| { + switch (self.devices[i].*) { + .cuda, .rocm, .oneapi => |*sv| { + const gpu = sv.front().*; + self.history.util[i].push(gpu.util_percent orelse 0); + const used = gpu.mem_used_bytes orelse 0; + const total = gpu.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(gpu.temperature orelse 0); + self.history.power[i].push(gpu.power_mw orelse 0); + }, + inline else => |*sv| { + const info = sv.front().*; + self.history.util[i].push(info.util_percent orelse 0); + const used = info.mem_used_bytes orelse 0; + const total = info.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(0); + self.history.power[i].push(0); + }, + } + } + } +}; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig new file mode 100644 index 00000000..1aea4778 --- /dev/null +++ b/bin/zml-smi/tui/image_cache.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); + +const ImageCache = @This(); + +pub var global: ImageCache = .{}; + +map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, + +pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { + self.map.deinit(allocator); +} + +pub fn load( + self: *ImageCache, + vx: *vaxis.Vaxis, + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + key: []const u8, + data: []const u8, +) void { + const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; + self.map.put(allocator, key, image) catch return; +} + +pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { + self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); + self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); + self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); + self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); + self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); + self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); +} + +pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { + return self.map.get(key); +} diff --git a/bin/zml-smi/tui/lib/compose.zig b/bin/zml-smi/tui/lib/compose.zig new file mode 100644 index 00000000..e3438762 --- /dev/null +++ b/bin/zml-smi/tui/lib/compose.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +// ── SurfaceBuilder ───────────────────────────────────────────────── + +pub const SurfaceBuilder = struct { + arena: std.mem.Allocator, + children: std.ArrayList(vxfw.SubSurface), + + pub fn add(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface) !void { + try self.addZ(row, col, surface, 0); + } + + pub fn addZ(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface, z_index: u8) !void { + try self.children.append(self.arena, .{ .origin = .{ .row = row, .col = col }, .surface = surface, .z_index = z_index }); + } + + pub fn finish(self: *SurfaceBuilder, size: vxfw.Size, wgt: vxfw.Widget) vxfw.Surface { + return .{ .size = size, .widget = wgt, .buffer = &.{}, .children = self.children.items }; + } +}; + +pub fn surfaceBuilder(arena: std.mem.Allocator) SurfaceBuilder { + return .{ .arena = arena, .children = .empty }; +} + +// ── Fluent widget wrappers ───────────────────────────────────────── + +pub fn center(arena: std.mem.Allocator, child: vxfw.Widget) std.mem.Allocator.Error!vxfw.Widget { + const c = try arena.create(vxfw.Center); + c.* = .{ .child = child }; + return c.widget(); +} + +pub fn sized(arena: std.mem.Allocator, child: vxfw.Widget, size: vxfw.Size) std.mem.Allocator.Error!vxfw.Widget { + const s = try arena.create(vxfw.SizedBox); + s.* = .{ .child = child, .size = size }; + return s.widget(); +} + +pub fn pad(arena: std.mem.Allocator, child: vxfw.Widget, padding: vxfw.Padding.PadValues) std.mem.Allocator.Error!vxfw.Widget { + const p = try arena.create(vxfw.Padding); + p.* = .{ .child = child, .padding = padding }; + return p.widget(); +} diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig new file mode 100644 index 00000000..a365ac4d --- /dev/null +++ b/bin/zml-smi/tui/lib/image.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const ui = @import("ui.zig"); +const Allocator = std.mem.Allocator; + +const Image = @This(); + +/// The loaded image to display. +image: vaxis.Image, +/// Desired height in cell rows. Width is computed to maintain aspect ratio. +/// If null, uses max constraint height (or 1 if unbounded). +rows: ?u16 = null, + +pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const target_rows = self.rows orelse ctx.max.height orelse 1; + if (target_rows == 0 or self.image.width == 0 or self.image.height == 0) { + return .{ + .size = .{ .width = 0, .height = 0 }, + .widget = ui.widget(self), + .buffer = &.{}, + .children = &.{}, + }; + } + + // Compute cell columns from image aspect ratio and terminal cell pixel size + const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; + const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); + const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + + const size: vxfw.Size = .{ .width = cols, .height = target_rows }; + var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); + surf.writeCell(0, 0, .{ + .image = .{ + .img_id = self.image.id, + .options = .{ + // Emit both rows and cols so that target bounds are unambiguous. + // Rows alone lets some terminals fall back to native pixel size. + .size = .{ .rows = target_rows, .cols = cols }, + }, + }, + }); + return surf; +} + +test Image { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + vxfw.DrawContext.init(.unicode); + + const img: Image = .{ + .image = .{ .id = 1, .width = 100, .height = 50 }, + .rows = 5, + }; + const w = ui.widget(&img); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{}, + .max = .{ .width = 40, .height = 10 }, + .cell_size = .{ .width = 10, .height = 20 }, + }; + + const surface = try w.draw(draw_ctx); + try std.testing.expectEqual(@as(u16, 5), surface.size.height); + // 5 rows * 20px = 100px height. scale = 100/50 = 2.0. width = 100*2 = 200px. cols = 200/10 = 20 + try std.testing.expectEqual(@as(u16, 20), surface.size.width); + // Image cell should be at (0, 0) + const cell = surface.buffer[0]; + try std.testing.expect(cell.image != null); + try std.testing.expectEqual(@as(u32, 1), cell.image.?.img_id); + // Both rows and cols must be emitted so terminals don't fall back to native pixel size. + const size = cell.image.?.options.size.?; + try std.testing.expectEqual(@as(?u16, 5), size.rows); + try std.testing.expectEqual(@as(?u16, 20), size.cols); +} + +test "refAllDecls" { + std.testing.refAllDecls(@This()); +} diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig new file mode 100644 index 00000000..6008d1e0 --- /dev/null +++ b/bin/zml-smi/tui/lib/ui.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +/// Creates a vxfw.Widget from any pointer whose type has a `draw` method +/// (and optionally a `handleEvent` method). +pub fn widget(ptr: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return self.draw(ctx); + } + }.f, + .eventHandler = if (@hasDecl(T, "handleEvent")) struct { + fn f(p: *anyopaque, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + const self: *T = @ptrCast(@alignCast(p)); + return self.handleEvent(ctx, event); + } + }.f else null, + }; +} + +/// Creates a vxfw.Widget that calls a specific draw function (for non-standard widget patterns +/// like MetricCard.drawContent or Chart.drawContent). +pub fn drawWidget(ptr: anytype, comptime drawFn: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return @call(.auto, drawFn, .{ self, ctx }); + } + }.f, + }; +} + +// ── DrawContext constraint helpers ────────────────────────────────── + +/// min = max = {w, h} — exact size. +pub fn fixedSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w, .height = h }, .{ .width = w, .height = h }); +} + +/// Width fixed (min = max = w), height unbounded. +pub fn fixedWidth(ctx: vxfw.DrawContext, w: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w }, .{ .width = w, .height = null }); +} + +/// max = {w, h}, min = default — bounded but unconstrained from below. +pub fn maxSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{}, .{ .width = w, .height = h }); +} + +// ── Drawing helpers ──────────────────────────────────────────────── + +/// Draw a single-line RichText with softwrap=false, overflow=clip, max width. +pub fn drawRichLine(ctx: vxfw.DrawContext, segments: []const vaxis.Cell.Segment, max_w: u16) std.mem.Allocator.Error!vxfw.Surface { + const rich: vxfw.RichText = .{ + .text = segments, + .softwrap = false, + .overflow = .clip, + }; + return rich.draw(ctx.withConstraints(.{}, .{ .width = max_w, .height = 1 })); +} + +pub const ImageCellSize = struct { + cols: u16, + rows: u16, +}; + +/// Compute the cell dimensions for an image scaled to a target row count, +/// maintaining aspect ratio based on the terminal's cell pixel size. +pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { + const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; + const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); + const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + return .{ .cols = img_cols, .rows = target_rows }; +} diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig new file mode 100644 index 00000000..993703c9 --- /dev/null +++ b/bin/zml-smi/tui/lib/utils.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +pub fn bytesToMb(val: ?u64) u64 { + return if (val) |v| v / (1024 * 1024) else 0; +} + +pub fn repeatStr(arena: std.mem.Allocator, char: []const u8, count: u16) std.mem.Allocator.Error![]const u8 { + const buf = try arena.alloc(u8, char.len * count); + for (0..count) |i| { + @memcpy(buf[i * char.len ..][0..char.len], char); + } + return buf; +} + +/// Maps values from [min_val, max_val] to [0, 100], clamped. +pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, max_val: u64) std.mem.Allocator.Error![]u8 { + const result = try arena.alloc(u8, raw.len); + const range = max_val -| min_val; + for (raw, 0..) |v, i| { + result[i] = if (v <= min_val) + 0 + else if (v >= max_val) + 100 + else if (range > 0) + @intCast((v - min_val) * 100 / range) + else + 0; + } + return result; +} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig new file mode 100644 index 00000000..86d4b7a3 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail.zig @@ -0,0 +1,28 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); +const gpu_detail = @import("detail/gpu_detail.zig"); +const neuron_detail = @import("detail/neuron_detail.zig"); +const tpu_detail = @import("detail/tpu_detail.zig"); + +const Detail = @This(); + +state: *const data.SystemState, +device_id: u16, +process_table: *ProcessTable = undefined, + +pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const id = self.device_id; + const dev = self.state.devices[id]; + const wgt = ui.widget(self); + + return switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + }; +} diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig new file mode 100644 index 00000000..93e8addb --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -0,0 +1,55 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const theme = @import("../../theme.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const compose = @import("../../lib/compose.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); + +pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { + return .{ + .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), + .style = theme.header_style, + .softwrap = false, + .overflow = .clip, + }; +} + +/// Creates a Chart pre-filled with history data and standard defaults. +pub fn historyChart( + ctx: vxfw.DrawContext, + state: *const data.SystemState, + id: u16, + history: anytype, + opts: struct { + title: []const u8, + value_label: []const u8, + info_line: ?[]const u8 = null, + y_min: u32 = 0, + y_max: u32 = 100, + y_unit: []const u8 = "%", + }, +) std.mem.Allocator.Error!Chart { + return .{ + .title = opts.title, + .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), + .value_label = opts.value_label, + .info_line = opts.info_line, + .y_min = opts.y_min, + .y_max = opts.y_max, + .y_unit = opts.y_unit, + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; +} + +/// Draws a page-level layout with standard margin (top=1, left/right=2) inside a full-width surface. +pub fn pageFrame(ctx: vxfw.DrawContext, layout: ColumnLayout, w: u16, parent_widget: vxfw.Widget) std.mem.Allocator.Error!vxfw.Surface { + const padded = try compose.pad(ctx.arena, ui.widget(&layout), .{ .top = 1, .left = 2, .right = 2 }); + var surf = try padded.draw(ui.fixedWidth(ctx, w)); + surf.widget = parent_widget; + return surf; +} diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig new file mode 100644 index 00000000..7884d6f8 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -0,0 +1,160 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + gpu: data.GpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const name_suffix: []const u8 = blk: { + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); + + // ── GPU Utilization + Memory charts ───────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "GPU Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), + .info_line = if (gpu.encoder_util_percent != null or gpu.decoder_util_percent != null) + try std.fmt.allocPrint(ctx.arena, "Enc {d}% Dec {d}%", .{ + gpu.encoder_util_percent orelse 0, gpu.decoder_util_percent orelse 0, + }) + else + null, + }); + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (gpu.mem_total_bytes) |total| if (total > 0) (gpu.mem_used_bytes orelse 0) * 100 / total else 0 else 0}), + .info_line = if (gpu.mem_bus_width) |bw| + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB ({d}-bit)", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), bw, + }) + else + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), + }), + }); + + // ── Temperature + Power charts ────────────────────────── + const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ + .title = "Temperature", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{gpu.fan_speed_percent orelse 0}), + .y_min = 20, + .y_max = 100, + .y_unit = "\u{00b0}C", + }); + + const power_limit_mw = gpu.power_limit_mw orelse 0; + const power_limit_w = power_limit_mw / 1000; + const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); + const power_watts = try ctx.arena.alloc(u64, power_raw.len); + for (power_raw, 0..) |p, j| { + power_watts[j] = p / 1000; + } + const y_max = if (power_limit_w > 0) power_limit_w else 1; + const power_chart: Chart = .{ + .title = "Power", + .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(gpu.power_mw orelse 0) / 1000}), + .info_line = if (power_limit_w > 0) + try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) + else + "N/A", + .y_min = 0, + .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), + .y_unit = "W", + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, + .min_child_width = 60, + .gap = 1, + }; + + // ── Clocks + PCIe ─────────────────────────────────────── + var clock_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try formatClockPair(ctx.arena, gpu.clock_graphics_mhz, gpu.clock_graphics_max_mhz) }); + if (gpu.clock_sm_mhz) |sm| { + try clock_lines.append(ctx.arena, .{ .label = "SM ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{sm}) }); + } + if (gpu.clock_soc_mhz) |soc| { + try clock_lines.append(ctx.arena, .{ .label = "SOC ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{soc}) }); + } + try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try formatClockPair(ctx.arena, gpu.clock_mem_mhz, gpu.clock_mem_max_mhz) }); + const clocks_card: MetricCard = .{ .title = "Clocks", .lines = clock_lines.items }; + + var pcie_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try formatPcieLink(ctx.arena, gpu.pcie_link_gen, gpu.pcie_link_width) }); + if (gpu.pcie_tx_kbps != null or gpu.pcie_rx_kbps != null) { + try pcie_lines.append(ctx.arena, .{ .label = "TX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_tx_kbps orelse 0) }); + try pcie_lines.append(ctx.arena, .{ .label = "RX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_rx_kbps orelse 0) }); + } + if (gpu.pcie_bandwidth_mbps) |bw| { + try pcie_lines.append(ctx.arena, .{ .label = "BW ", .value = try std.fmt.allocPrint(ctx.arena, " {d} Mb/s", .{bw}) }); + } + const pcie_card: MetricCard = .{ .title = "PCIe", .lines = pcie_lines.items }; + + const cards_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&clocks_card), ui.widget(&pcie_card) }, + .min_child_width = 30, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&cards_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} + +fn formatClockPair(arena: std.mem.Allocator, current: ?u64, max: ?u64) std.mem.Allocator.Error![]const u8 { + if (current == null and max == null) return " N/A"; + return std.fmt.allocPrint(arena, " {d} / {d} MHz", .{ current orelse 0, max orelse 0 }); +} + +fn formatPcieLink(arena: std.mem.Allocator, gen: ?u64, width: ?u64) std.mem.Allocator.Error![]const u8 { + if (gen == null and width == null) return " N/A"; + return std.fmt.allocPrint(arena, " Gen{d} x{d}", .{ gen orelse 0, width orelse 0 }); +} + +fn formatBandwidth(arena: std.mem.Allocator, kbps: u64) std.mem.Allocator.Error![]const u8 { + if (kbps >= 1_000_000) { + const gbps = @as(f64, @floatFromInt(kbps)) / 1_000_000.0; + return std.fmt.allocPrint(arena, " {d:.1} GB/s", .{gbps}); + } else if (kbps >= 1_000) { + const mbps = @as(f64, @floatFromInt(kbps)) / 1_000.0; + return std.fmt.allocPrint(arena, " {d:.1} MB/s", .{mbps}); + } else { + return std.fmt.allocPrint(arena, " {d} KB/s", .{kbps}); + } +} diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig new file mode 100644 index 00000000..e0dc3bcd --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -0,0 +1,83 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + nc: data.NeuronInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const name_suffix: []const u8 = if (nc.driver_version) |v| + try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) + else + ""; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Core Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{nc.util_percent orelse 0}), + }); + + const mem_used = nc.mem_used_bytes orelse 0; + const mem_total = nc.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(nc.mem_used_bytes), utils.bytesToMb(nc.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── HBM Breakdown ─────────────────────────────────────── + const fmt_mb = struct { + fn f(arena: std.mem.Allocator, val: ?u64) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, " {d} MB", .{utils.bytesToMb(val)}); + } + }.f; + const hbm_card: MetricCard = .{ + .title = "HBM Breakdown", + .lines = &.{ + .{ .label = "Tensors ", .value = try fmt_mb(ctx.arena, nc.nc_tensors) }, + .{ .label = "Constants ", .value = try fmt_mb(ctx.arena, nc.nc_constants) }, + .{ .label = "Model Code ", .value = try fmt_mb(ctx.arena, nc.nc_model_code) }, + .{ .label = "Shared Scratch", .value = try fmt_mb(ctx.arena, nc.nc_shared_scratchpad) }, + .{ .label = "Nonshared ", .value = try fmt_mb(ctx.arena, nc.nc_nonshared_scratchpad) }, + .{ .label = "Runtime ", .value = try fmt_mb(ctx.arena, nc.nc_runtime) }, + .{ .label = "Driver ", .value = try fmt_mb(ctx.arena, nc.nc_driver) }, + .{ .label = "DMA Rings ", .value = try fmt_mb(ctx.arena, nc.nc_dma_rings) }, + .{ .label = "Collectives ", .value = try fmt_mb(ctx.arena, nc.nc_collectives) }, + .{ .label = "Notifications ", .value = try fmt_mb(ctx.arena, nc.nc_notifications) }, + .{ .label = "Uncategorized ", .value = try fmt_mb(ctx.arena, nc.nc_uncategorized) }, + }, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&hbm_card), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig new file mode 100644 index 00000000..ca6d3b66 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + tp: data.TpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Duty Cycle", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{tp.util_percent orelse 0}), + }); + + const mem_used = tp.mem_used_bytes orelse 0; + const mem_total = tp.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(tp.mem_used_bytes), utils.bytesToMb(tp.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig new file mode 100644 index 00000000..e91253bd --- /dev/null +++ b/bin/zml-smi/tui/pages/overview.zig @@ -0,0 +1,119 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const image_cache = @import("../image_cache.zig"); +const ColumnLayout = @import("../widgets/column_layout.zig"); +const common = @import("detail/detail_common.zig"); +const DeviceCard = @import("../widgets/device_card.zig"); +const Logo = @import("../widgets/logo.zig"); +const InfoLines = @import("../widgets/info_lines.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); + +const Overview = @This(); + +const two_col_breakpoint: u16 = 140; +const narrow_breakpoint: u16 = 80; +const max_info_width: u16 = 50; +/// Total width of the wide banner: logo box (incl. left margin) + info lines + right margin. +pub const host_line_width: u16 = (Logo.logo_width + 6) + max_info_width + 2; + +state: *const data.SystemState, +device_cards: []DeviceCard = &.{}, +process_table: ?*ProcessTable = null, // maybe null if we decide not to print the process table in print mode +viewing_device: *?u16 = undefined, +use_braille: bool = false, + +pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u16) !Overview { + const count = state.deviceCount(); + const cards = try allocator.alloc(DeviceCard, count); + for (cards, 0..) |*card, i| { + card.* = .{ + .device_id = @intCast(i), + .state = state, + }; + } + return .{ + .state = state, + .device_cards = cards, + .process_table = process_table, + .viewing_device = viewing_device, + }; +} + +pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { + allocator.free(self.device_cards); +} + +fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const info_lines: InfoLines = .{ .state = self.state }; + + const children = [2]vxfw.Widget{ + try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = Logo.logo_height }), + ui.widget(&info_lines), + }; + const layout: ColumnLayout = .{ .children = &children, .gap = 1 }; + return ui.widget(&layout).draw(ui.fixedWidth(ctx, content_w)); +} + +fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const info_lines: InfoLines = .{ .state = self.state }; + + const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin + const info_max_w = @min(content_w -| logo_box_w, max_info_width); + const banner_h = @max(Logo.logo_height, InfoLines.entry_count); + + const flex_items = [2]vxfw.FlexItem{ + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = logo_box_w, .height = banner_h }), .flex = 0 }, + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&info_lines)), .{ .width = info_max_w, .height = banner_h }), .flex = 0 }, + }; + const sized_banner = try compose.sized(ctx.arena, (vxfw.FlexRow{ .children = &flex_items }).widget(), .{ .width = content_w, .height = banner_h }); + return sized_banner.draw(ui.fixedWidth(ctx, content_w)); +} + +pub fn draw(self: *Overview, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const narrow = w < narrow_breakpoint; + const content_w: u16 = w -| 4; + const content_ctx = ui.fixedWidth(ctx, content_w); + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 1; // top margin + + // ── Banner (Logo + Info Lines) ───────────────────────── + const banner_surf = try if (narrow) self.drawNarrowBanner(ctx, content_w) else self.drawWideBanner(ctx, content_w + 2); + try sb.add(row, if (narrow) @as(i17, 2) else 0, banner_surf); + row += @intCast(banner_surf.size.height + 1); + + // ── Device Grid ───────────────────────────────────────── + for (self.device_cards) |*card| { + card.use_braille = self.use_braille; + card.viewing_device = self.viewing_device; + } + const card_widgets = try ctx.arena.alloc(vxfw.Widget, self.device_cards.len); + for (self.device_cards, card_widgets) |*card, *cw| { + cw.* = ui.widget(card); + } + const device_grid: ColumnLayout = .{ + .children = card_widgets, + .min_child_width = two_col_breakpoint / 2, + .col_gap = 2, + .gap = 1, + }; + const grid_surf = try ui.widget(&device_grid).draw(content_ctx); + try sb.add(row, 2, grid_surf); + row += @intCast(grid_surf.size.height + 1); + + // ── Process Table ──────────────────────────────────────── + if (self.process_table) |pt| { + const pt_surf = try ui.widget(pt).draw(content_ctx); + try sb.add(row, 2, pt_surf); + row += @intCast(pt_surf.size.height); + } + + return sb.finish(.{ .width = w, .height = @intCast(row) }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig new file mode 100644 index 00000000..238534cd --- /dev/null +++ b/bin/zml-smi/tui/print.zig @@ -0,0 +1,91 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const image_cache = @import("image_cache.zig"); +const Overview = @import("pages/overview.zig"); + +fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.Winsize) !void { + vx.screen.deinit(allocator); + vx.screen = try vaxis.Screen.init(allocator, winsize); + vx.screen.width_method = vx.caps.unicode; + vx.screen_last.deinit(allocator); + vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); +} + +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var tty = try vaxis.Tty.init(io, &tty_buffer); + defer tty.deinit(); + + var vx = try vaxis.init(io, allocator, env_map, .{}); + defer { + // Reset terminal modes without clearing screen content. + vx.resetModes(tty.writer()) catch {}; + var buf: [256]u8 = undefined; + var discarding: std.Io.Writer.Discarding = .init(&buf); + vx.deinit(allocator, &discarding.writer); + } + + const ws = try tty.getWinsize(); + try silentResize(&vx, allocator, ws); + + { + const EventLoop = vaxis.Loop(vxfw.Event); + var loop: EventLoop = .init(io, &tty, &vx); + try loop.start(); + defer loop.stop(); + + // queryTerminal sends cursor home (\x1b[H) for capability detection. + // Save/restore cursor so we stay at the current position. + const writer = tty.writer(); + try writer.writeAll("\x1b7"); // DECSC: save cursor + try writer.flush(); + try vx.queryTerminal(writer, .fromSeconds(1)); + try writer.writeAll("\x1b8"); // DECRC: restore cursor + try writer.flush(); + } + + image_cache.global.loadAll(&vx, allocator, tty.writer()); + defer image_cache.global.deinit(allocator); + + vxfw.DrawContext.init(vx.screen.width_method); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); + + var viewing_device: ?u16 = null; + var overview = try Overview.init(allocator, state, null, &viewing_device); + defer overview.deinit(allocator); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{ .width = content_w }, + .max = .{ .width = content_w, .height = null }, + .cell_size = .{ + .width = if (vx.screen.width > 0) vx.screen.width_pix / vx.screen.width else 0, + .height = if (vx.screen.height > 0) vx.screen.height_pix / vx.screen.height else 0, + }, + }; + + const surface = try overview.draw(draw_ctx); + + try silentResize(&vx, allocator, .{ + .rows = surface.size.height, + .cols = content_w, + .x_pixel = ws.x_pixel, + .y_pixel = ws.y_pixel, + }); + + const win = vx.window(); + const root_win = win.child(.{ + .width = surface.size.width, + .height = surface.size.height, + }); + surface.render(root_win, ui.widget(&overview)); + + try vx.prettyPrint(tty.writer()); +} diff --git a/bin/zml-smi/tui/theme.zig b/bin/zml-smi/tui/theme.zig new file mode 100644 index 00000000..e037f67f --- /dev/null +++ b/bin/zml-smi/tui/theme.zig @@ -0,0 +1,50 @@ +const vaxis = @import("vaxis"); + +const Color = vaxis.Cell.Color; +const Style = vaxis.Cell.Style; + +/// Returns a color on a green → yellow → red gradient based on percentage (0-100). +pub fn colorForPercent(pct: u8) Color { + if (pct <= 50) { + // Green (#00cc66) → Yellow (#ffcc00) + const t: u16 = @as(u16, pct) * 255 / 50; + return .{ .rgb = .{ @intCast(t), 204, @intCast(102 -| (t * 102 / 255)) } }; + } else if (pct <= 80) { + // Yellow (#ffcc00) → Orange (#ff6600) + const t: u16 = (@as(u16, pct) - 50) * 255 / 30; + return .{ .rgb = .{ 255, @intCast(204 -| (t * 102 / 255)), 0 } }; + } else { + // Orange (#ff6600) → Red (#ff2200) + const t: u16 = (@as(u16, pct) - 80) * 255 / 20; + return .{ .rgb = .{ 255, @intCast(102 -| (t * 68 / 255)), 0 } }; + } +} + +/// Returns a color for temperature (20-100°C range). +pub fn colorForTemp(temp_c: u16) Color { + if (temp_c <= 20) { + return colorForPercent(0); + } + if (temp_c >= 100) { + return colorForPercent(100); + } + const pct: u8 = @intCast((@as(u32, temp_c) - 20) * 100 / 80); + return colorForPercent(pct); +} + +// ── Style Constants ────────────────────────────────────────────── + +pub const accent: Color = .{ .rgb = .{ 120, 180, 255 } }; +pub const accent_secondary: Color = .{ .rgb = .{ 200, 160, 255 } }; +pub const dim: Color = .{ .rgb = .{ 100, 100, 120 } }; +pub const text_primary: Color = .{ .rgb = .{ 230, 230, 240 } }; +pub const text_secondary: Color = .{ .rgb = .{ 160, 160, 175 } }; +pub const surface_border: Color = .{ .rgb = .{ 70, 70, 90 } }; +pub const gauge_empty: Color = .{ .rgb = .{ 50, 50, 65 } }; + +pub const header_style: Style = .{ .bold = true, .fg = accent }; +pub const title_style: Style = .{ .bold = true, .fg = accent_secondary }; +pub const label_style: Style = .{ .fg = text_secondary }; +pub const value_style: Style = .{ .bold = true, .fg = text_primary }; +pub const border_style: Style = .{ .fg = surface_border }; +pub const dim_style: Style = .{ .fg = dim }; diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig new file mode 100644 index 00000000..74b3796b --- /dev/null +++ b/bin/zml-smi/tui/top.zig @@ -0,0 +1,168 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const compose = @import("lib/compose.zig"); +const image_cache = @import("image_cache.zig"); +const Logo = @import("widgets/logo.zig"); +const ProcessTable = @import("widgets/process_table.zig"); +const Overview = @import("pages/overview.zig"); +const Detail = @import("pages/detail.zig"); +const StatusLine = @import("widgets/status_line.zig"); + +const min_width: u16 = 45; +const min_height: u16 = 24; + +const ScrollState = struct { + row: i17 = 0, + col: i17 = 0, + + pub fn reset(self: *ScrollState) void { + self.row = 0; + self.col = 0; + } + + pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { + self.row = @max(0, self.row + dr); + self.col = @max(0, self.col + dc); + } + + pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { + const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); + const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); + self.row = std.math.clamp(self.row, 0, max_row); + self.col = std.math.clamp(self.col, 0, max_col); + } +}; + +const Model = struct { + allocator: std.mem.Allocator, + state: *data.SystemState, + viewing_device: ?u16 = null, + prev_viewing_device: ?u16 = null, + scroll: ScrollState = .{}, + vx: *vaxis.Vaxis, + tty: *vaxis.Tty, + process_table: ProcessTable = .{}, + overview: Overview = undefined, + + fn deinit(self: *Model) void { + self.process_table.deinit(self.allocator); + self.overview.deinit(self.allocator); + image_cache.global.deinit(self.allocator); + } + + pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .init => { + self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); + self.overview.use_braille = true; + image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); + try ctx.tick(1000, ui.widget(self)); + }, + .tick => { + self.state.recordHistory(); + try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); + ctx.redraw = true; + }, + .key_press => |key| { + if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { + ctx.quit = true; + return; + } + if (self.viewing_device != null) { + if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { + self.viewing_device = null; + return ctx.consumeAndRedraw(); + } + } else { + if (key.matches('v', .{})) { + self.overview.use_braille = !self.overview.use_braille; + return ctx.consumeAndRedraw(); + } + } + if (key.matches(vaxis.Key.down, .{})) { + self.scroll.scrollBy(1, 0); + } else if (key.matches(vaxis.Key.up, .{})) { + self.scroll.scrollBy(-1, 0); + } else if (key.matches(vaxis.Key.right, .{})) { + self.scroll.scrollBy(0, 1); + } else if (key.matches(vaxis.Key.left, .{})) { + self.scroll.scrollBy(0, -1); + } else return; + return ctx.consumeAndRedraw(); + }, + .mouse => |mouse| { + switch (mouse.button) { + .wheel_up => self.scroll.scrollBy(-3, 0), + .wheel_down => self.scroll.scrollBy(3, 0), + .wheel_left => self.scroll.scrollBy(0, 3), + .wheel_right => self.scroll.scrollBy(0, -3), + else => return, + } + return ctx.consumeAndRedraw(); + }, + else => {}, + } + } + + pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const screen = ctx.max.size(); + const content_w = @max(screen.width, min_width); + + if (self.viewing_device != self.prev_viewing_device) { + self.scroll.reset(); + self.process_table.resetScroll(); + self.prev_viewing_device = self.viewing_device; + } + + self.process_table.prepare(self.state, self.viewing_device); + + var content_surf: vxfw.Surface = undefined; + + if (self.viewing_device) |dev_id| { + const detail: Detail = .{ + .state = self.state, + .device_id = dev_id, + .process_table = &self.process_table, + }; + content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); + } else { + content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); + } + + content_surf.size.height = @max(content_surf.size.height, min_height); + + const content_h = screen.height -| 1; + self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); + + const status_line: StatusLine = .{ + .viewing_device = self.viewing_device, + .use_braille = self.overview.use_braille, + }; + const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); + + var sb = compose.surfaceBuilder(ctx.arena); + try sb.add(-self.scroll.row, -self.scroll.col, content_surf); + try sb.addZ(@intCast(content_h), 0, status_surf, 1); + return sb.finish(screen, ui.widget(self)); + } +}; + +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); + defer app.deinit(); + + var model: Model = .{ + .allocator = allocator, + .state = state, + .vx = &app.vx, + .tty = &app.tty, + }; + defer model.deinit(); + + try app.run(ui.widget(&model), .{}); +} diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig new file mode 100644 index 00000000..015e3ce2 --- /dev/null +++ b/bin/zml-smi/tui/tui.zig @@ -0,0 +1,3 @@ +pub const data = @import("data.zig"); +pub const print = @import("print.zig"); +pub const top = @import("top.zig"); diff --git a/bin/zml-smi/tui/widgets/braille_chart.zig b/bin/zml-smi/tui/widgets/braille_chart.zig new file mode 100644 index 00000000..498739d1 --- /dev/null +++ b/bin/zml-smi/tui/widgets/braille_chart.zig @@ -0,0 +1,76 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); + +const ui = @import("../lib/ui.zig"); + +const BrailleChart = @This(); + +values: []const u8, // 0-100 +height: u16 = 2, + +pub const chart_bg: vaxis.Cell.Color = .{ .rgb = .{ 30, 30, 40 } }; + +/// Braille dot bit positions: [row_in_cell (0-3)][col_in_cell (0-1)] +/// Each braille character is a 2x4 dot grid encoded as U+2800 + pattern byte. +pub const dot_bits = [4][2]u8{ + .{ 0x01, 0x08 }, // row 0 + .{ 0x02, 0x10 }, // row 1 + .{ 0x04, 0x20 }, // row 2 + .{ 0x40, 0x80 }, // row 3 +}; + +pub fn draw(self: *const BrailleChart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 40; + var surface = try vxfw.Surface.init(ctx.arena, ui.widget(self), .{ .width = w, .height = self.height }); + try self.renderTo(ctx.arena, &surface, 0, 0, w); + return surface; +} + +/// Render braille data into an existing surface at the given offset. +pub fn renderTo(self: *const BrailleChart, arena: std.mem.Allocator, surface: *vxfw.Surface, col_offset: u16, row_offset: u16, w: u16) std.mem.Allocator.Error!void { + const h = self.height; + const dot_rows: u32 = @as(u32, h) * 4; + const dot_cols: u32 = @as(u32, w) * 2; + const data_len: u32 = @intCast(self.values.len); + + for (0..w) |cell_col| { + var max_val: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + max_val = @max(max_val, self.values[@intCast(dot_x + data_len - dot_cols)]); + } + } + const color = theme.colorForPercent(max_val); + + for (0..h) |cell_row| { + var pattern: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + const val = self.values[@intCast(dot_x + data_len - dot_cols)]; + const fill: u32 = @as(u32, val) * dot_rows / 100; + const threshold: u32 = dot_rows - fill; + for (0..4) |dr| { + const abs_row: u32 = @as(u32, @intCast(cell_row)) * 4 + @as(u32, @intCast(dr)); + if (abs_row >= threshold) { + pattern |= dot_bits[dr][dx]; + } + } + } + } + + const buf = try arena.alloc(u8, 3); + buf[0] = 0xE2; + buf[1] = 0xA0 + (pattern >> 6); + buf[2] = 0x80 + (pattern & 0x3F); + + surface.writeCell(col_offset + @as(u16, @intCast(cell_col)), row_offset + @as(u16, @intCast(cell_row)), .{ + .char = .{ .grapheme = buf, .width = 1 }, + .style = .{ .fg = color, .bg = chart_bg }, + }); + } + } +} diff --git a/bin/zml-smi/tui/widgets/chart.zig b/bin/zml-smi/tui/widgets/chart.zig new file mode 100644 index 00000000..5421cd1f --- /dev/null +++ b/bin/zml-smi/tui/widgets/chart.zig @@ -0,0 +1,141 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const TitledBorder = @import("titled_border.zig"); +const BrailleChart = @import("braille_chart.zig"); + +const Chart = @This(); + +// Layout constants for rows below the chart data area +const x_axis_h: u16 = 1; // x-axis line +const time_labels_h: u16 = 1; // time labels below x-axis +const info_line_h: u16 = 1; // info text below time labels + +title: []const u8, +data: []const u8, // 0-100 normalized +value_label: ?[]const u8 = null, +info_line: ?[]const u8 = null, +y_min: u32 = 0, +y_max: u32 = 100, +y_unit: []const u8 = "%", +chart_height: u16 = 6, +tui_refresh_rate: u16, + +pub fn draw(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .value_label = self.value_label, + }; + return tb.draw(ctx); +} + +fn drawContent(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 40; + const content_h: u16 = self.chart_height + x_axis_h + time_labels_h + info_line_h; + + // ── Y-axis label widths ────────────────────────────────── + const y_labels = [3]struct { row: u16, val: u32 }{ + .{ .row = 0, .val = self.y_max }, + .{ .row = self.chart_height / 2, .val = (self.y_min + self.y_max) / 2 }, + .{ .row = self.chart_height, .val = self.y_min }, + }; + var y_strs: [3][]const u8 = undefined; + var y_str_widths: [3]u16 = undefined; + var y_label_w: u16 = 0; + for (&y_strs, &y_str_widths, y_labels) |*s, *sw, lbl| { + s.* = try std.fmt.allocPrint(ctx.arena, "{d}{s}", .{ lbl.val, self.y_unit }); + sw.* = @intCast(ctx.stringWidth(s.*)); + y_label_w = @max(y_label_w, sw.*); + } + + // ── Column layout ──────────────────────────────────────── + const chart_start: u16 = y_label_w + 1; + const chart_w: u16 = if (max_w > chart_start) max_w - chart_start else 0; + + var surface = try vxfw.Surface.init(ctx.arena, ui.drawWidget(self, drawContent), .{ .width = max_w, .height = content_h }); + if (chart_w == 0) { + return surface; + } + + // ── Y-axis labels + axis line ──────────────────────────── + for (y_strs, y_str_widths, y_labels) |text, sw, lbl| { + writeStr(&surface, y_label_w - sw, lbl.row, text, theme.dim_style); + } + for (0..self.chart_height + 1) |r| { + const row: u16 = @intCast(r); + const ch: []const u8 = if (row == self.chart_height) + "\u{2514}" + else + "\u{2502}"; + surface.writeCell(y_label_w, row, .{ + .char = .{ .grapheme = ch, .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── Braille data area ──────────────────────────────────── + const braille: BrailleChart = .{ .values = self.data, .height = self.chart_height }; + try braille.renderTo(ctx.arena, &surface, chart_start, 0, chart_w); + + // ── X-axis line ────────────────────────────────────────── + const tick_mid: u16 = chart_w / 2; + const tick_right: u16 = chart_w -| 1; + for (0..chart_w) |c| { + const col: u16 = @intCast(c); + const is_tick = (col == 0 or col == tick_mid or col == tick_right); + surface.writeCell(chart_start + col, self.chart_height, .{ + .char = .{ .grapheme = if (is_tick) "\u{252C}" else "\u{2500}", .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── X-axis time labels ─────────────────────────────────── + const total_ms: u32 = @as(u32, chart_w) * 2 * @as(u32, self.tui_refresh_rate); + const total_s: u16 = @intCast(std.math.divCeil(u32, total_ms, 1000) catch unreachable); + const mid_s: u16 = @intCast(std.math.divCeil(u32, total_ms / 2, 1000) catch unreachable); + const label_row = self.chart_height + x_axis_h; + const left_str = try fmtTime(ctx.arena, total_s); + const mid_str = try fmtTime(ctx.arena, mid_s); + + writeStr(&surface, chart_start, label_row, left_str, theme.dim_style); + + const mid_w: u16 = @intCast(ctx.stringWidth(mid_str)); + writeStr(&surface, chart_start + tick_mid -| (mid_w / 2), label_row, mid_str, theme.dim_style); + + const right_str: []const u8 = "now"; + const right_w: u16 = @intCast(ctx.stringWidth(right_str)); + writeStr(&surface, chart_start + tick_right + 1 -| right_w, label_row, right_str, theme.dim_style); + + // ── Info line ──────────────────────────────────────────── + if (self.info_line) |info| { + writeStr(&surface, chart_start, self.chart_height + x_axis_h + time_labels_h, info, theme.dim_style); + } + + return surface; +} + +fn writeStr(surface: *vxfw.Surface, col: u16, row: u16, text: []const u8, style: vaxis.Cell.Style) void { + var c = col; + var i: usize = 0; + while (i < text.len) { + const len = std.unicode.utf8ByteSequenceLength(text[i]) catch 1; + const end = @min(i + len, text.len); + surface.writeCell(c, row, .{ + .char = .{ .grapheme = text[i..end], .width = 1 }, + .style = style, + }); + c += 1; + i = end; + } +} + +fn fmtTime(arena: std.mem.Allocator, seconds: u16) std.mem.Allocator.Error![]const u8 { + if (seconds >= 120) { + return std.fmt.allocPrint(arena, "-{d}m", .{seconds / 60}); + } else { + return std.fmt.allocPrint(arena, "-{d}s", .{seconds}); + } +} diff --git a/bin/zml-smi/tui/widgets/column_layout.zig b/bin/zml-smi/tui/widgets/column_layout.zig new file mode 100644 index 00000000..f307d40c --- /dev/null +++ b/bin/zml-smi/tui/widgets/column_layout.zig @@ -0,0 +1,64 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); + +const ColumnLayout = @This(); + +children: []const vxfw.Widget, +gap: u16 = 0, +col_gap: u16 = 0, +/// When set, children wrap into a grid with this minimum column width. +/// When null (default), behaves as a simple vertical stack. +min_child_width: ?u16 = null, + +pub fn draw(self: *const ColumnLayout, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const n = self.children.len; + if (n == 0) { + return vxfw.Surface.init(ctx.arena, ui.widget(self), ctx.min); + } + + const available_w = ctx.max.width orelse 80; + const cols: u16 = if (self.min_child_width) |mcw| + @intCast(@max(1, @min(n, available_w / @max(mcw, 1)))) + else + 1; + + var sb = compose.surfaceBuilder(ctx.arena); + var row_y: u16 = 0; + var max_width: u16 = 0; + var i: usize = 0; + + while (i < n) { + if (i > 0) { + row_y += self.gap; + } + + const row_end = @min(i + cols, n); + var row_h: u16 = 0; + + for (i..row_end) |j| { + const col_idx: u16 = @intCast(j - i); + + const col_x, const w = if (cols > 1) blk: { + const total_col_gaps = @as(u16, cols - 1) * self.col_gap; + const usable_w = available_w -| total_col_gaps; + const child_w = usable_w / cols; + const x = col_idx * (child_w + self.col_gap); + break :blk .{ x, if (col_idx == cols - 1) available_w - x else child_w }; + } else .{ 0, available_w }; + + const surf = try self.children[j].draw(ui.fixedWidth(ctx, w)); + try sb.add(row_y, col_x, surf); + max_width = @max(max_width, col_x + surf.size.width); + row_h = @max(row_h, surf.size.height); + } + + row_y += row_h; + i = row_end; + } + + const result_w = if (ctx.max.width) |mw| @min(@max(max_width, ctx.min.width), mw) else @max(max_width, ctx.min.width); + return sb.finish(.{ .width = result_w, .height = row_y }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig new file mode 100644 index 00000000..c7f7ac38 --- /dev/null +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -0,0 +1,148 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); +const image_cache = @import("../image_cache.zig"); +const MetricCard = @import("metric_card.zig"); + +const DeviceCard = @This(); + +device_id: u16 = 0, +highlighted: bool = false, + +state: *const data.SystemState = undefined, +use_braille: bool = false, +viewing_device: *?u16 = undefined, + +pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .mouse => |mouse| { + if (mouse.type == .press and mouse.button == .left) { + self.viewing_device.* = self.device_id; + return ctx.consumeAndRedraw(); + } + }, + .mouse_enter => { + self.highlighted = true; + try ctx.setMouseShape(.pointer); + return ctx.consumeAndRedraw(); + }, + .mouse_leave => { + self.highlighted = false; + try ctx.setMouseShape(.default); + ctx.redraw = true; + }, + else => {}, + } +} + +pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const i = self.device_id; + const dev = self.state.devices[i]; + const target: data.Target = std.meta.activeTag(dev.*); + const cf = switch (dev.*) { + inline else => |*sv| blk: { + const f = sv.front().*; + break :blk .{ + .name = f.name orelse "Unknown", + .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), + .mem_used = f.mem_used_bytes orelse 0, + .mem_total = f.mem_total_bytes orelse 0, + }; + }, + }; + const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; + + const device_label, const util_label = switch (target) { + .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, + .neuron => .{ "NC", "Core" }, + .tpu => .{ "TPU", "Duty" }, + }; + const driver_ver: []const u8 = switch (dev.*) { + .cuda => |*sv| blk: { + const gpu = sv.front(); + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }, + .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + else => "", + }; + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); + + const dev_suffix = switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| blk: { + const gpu = sv.front().*; + const power = (gpu.power_mw orelse 0) / 1000; + const power_limit = (gpu.power_limit_mw orelse 0) / 1000; + + const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); + const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); + + const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); + break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ + gpu.temperature orelse 0, pad, power_str, + }); + }, + else => "", + }; + + const mem_used_mb = utils.bytesToMb(cf.mem_used); + const mem_total_mb = utils.bytesToMb(cf.mem_total); + const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); + const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); + + const mem_pad = try utils.repeatStr(ctx.arena, " ", @intCast(mem_total_str.len -| mem_used_str.len)); + const mem_suffix = try std.fmt.allocPrint(ctx.arena, "{s}{s} / {s} MB", .{ + mem_pad, mem_used_str, mem_total_str, + }); + + var card: MetricCard = .{ + .title = card_title, + .title_image = image_cache.global.get(switch (dev.*) { + .cuda => "gpu_cuda", + .rocm => "gpu_rocm", + .oneapi => "gpu_oneapi", + .neuron => "gpu_neuron", + .tpu => "gpu_tpu", + }), + .title_right = if (self.state.devices[i].isRemote()) "remote" else null, + .cell_size = ctx.cell_size, + .highlighted = self.highlighted, + }; + + const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); + const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); + const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); + const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); + + const chart_entries = [2]MetricCard.ChartEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, + }; + const gauge_entries = [2]MetricCard.GaugeEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, + }; + + if (self.use_braille) { + card.charts = &chart_entries; + } else { + card.gauges = &gauge_entries; + } + + var card_surf = try card.draw(ctx); + card_surf.widget = ui.widget(self); + return card_surf; +} diff --git a/bin/zml-smi/tui/widgets/gauge.zig b/bin/zml-smi/tui/widgets/gauge.zig new file mode 100644 index 00000000..865a45f5 --- /dev/null +++ b/bin/zml-smi/tui/widgets/gauge.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); + +const RichText = vxfw.RichText; + +const Gauge = @This(); + +value: u8, // 0-100 +label: ?[]const u8 = null, +suffix: ?[]const u8 = null, +/// Minimum label space to reserve (for aligning gauges with different label widths). +label_reserve: u16 = 0, +/// Minimum suffix space to reserve (for aligning gauges with different suffix widths). +suffix_reserve: u16 = 0, + +pub const sep_w: u16 = 3; // " " before suffix + +pub fn draw(self: *const Gauge, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 30; + const width: u16 = @max(ctx.min.width, max_w); + + // Label width (use reserve for alignment if set) + var label_w: u16 = 0; + if (self.label) |label| { + label_w = @intCast(ctx.stringWidth(label)); + label_w += 1; // trailing space + } + const effective_label_w = @max(label_w, self.label_reserve); + + // Percentage text: " XX%" + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{self.value}); + const pct_width: u16 = @intCast(ctx.stringWidth(pct_str)); + + // Suffix area (text + separator); use reserve for bar calculation, actual for rendering + const actual_suffix_area: u16 = if (self.suffix) |s| @as(u16, @intCast(ctx.stringWidth(s))) + sep_w else 0; + const effective_suffix_area = @max(actual_suffix_area, self.suffix_reserve); + + // Reserved: label + "[" + "]" + pct + suffix_area + const reserved = effective_label_w + pct_width + 2 + effective_suffix_area; // 2 for [ ] + const bar_width = width -| reserved; + const filled: u16 = @intCast(@as(u32, bar_width) * @as(u32, @min(self.value, 100)) / 100); + const empty_count = bar_width - filled; + + var spans: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Label (padded to effective_label_w for alignment) + if (self.label) |label| { + try spans.append(ctx.arena, .{ .text = label, .style = theme.label_style }); + const pad = effective_label_w -| label_w; + const pad_str = try utils.repeatStr(ctx.arena, " ", pad + 1); // +1 for trailing space + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.label_style }); + } + + // Opening bracket + try spans.append(ctx.arena, .{ .text = "[", .style = theme.dim_style }); + + // Filled blocks — per-block gradient color + for (0..filled) |i| { + const pos_pct: u8 = if (bar_width > 1) + @intCast(@as(u32, @intCast(i)) * 100 / (@as(u32, bar_width) - 1)) + else + 0; + try spans.append(ctx.arena, .{ + .text = "\u{2588}", + .style = .{ .fg = theme.colorForPercent(pos_pct) }, + }); + } + + // Empty blocks + if (empty_count > 0) { + const empty_str = try utils.repeatStr(ctx.arena, "\u{2591}", empty_count); + try spans.append(ctx.arena, .{ .text = empty_str, .style = .{ .fg = theme.gauge_empty } }); + } + + // Closing bracket (no space — pct% collides with bar) + try spans.append(ctx.arena, .{ .text = "]", .style = theme.dim_style }); + + // Percentage text + try spans.append(ctx.arena, .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(self.value) } }); + + // Suffix (right-aligned) + if (self.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const pad = effective_suffix_area -| suffix_text_w; + if (pad > 0) { + const pad_str = try utils.repeatStr(ctx.arena, " ", pad); + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.dim_style }); + } + try spans.append(ctx.arena, .{ .text = suffix, .style = theme.dim_style }); + } + + const rich: RichText = .{ + .text = spans.items, + .softwrap = false, + .overflow = .clip, + .width_basis = .parent, + }; + return rich.draw(ui.fixedSize(ctx, width, 1)); +} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig new file mode 100644 index 00000000..fcf1ae0d --- /dev/null +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -0,0 +1,112 @@ +const std = @import("std"); + +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("../data.zig"); +const compose = @import("../lib/compose.zig"); +const ui = @import("../lib/ui.zig"); +const theme = @import("../theme.zig"); +const stamp = @import("bazel/stamp"); + +const InfoLines = @This(); + +state: *const data.SystemState, + +const labels = [_][]const u8{ + "zml-smi ", + "Hostname ", + "Kernel ", + "CPU ", + "Cores ", + "Memory ", + "Uptime ", + "Load ", + "Devices ", +}; + +pub const entry_count: u16 = labels.len; + +const Entry = struct { + label: []const u8, + value: []const u8, +}; + +pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const host = self.state.host.front().*; + + const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); + const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); + const cpu_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.cpu_name orelse "Unknown"}); + const uptime_str = try formatUptime(ctx.arena, host.uptime_seconds orelse 0); + const cores_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{host.cpu_cores orelse 0}); + const load_str = try std.fmt.allocPrint(ctx.arena, "{d:.2} / {d:.2} / {d:.2}", .{ + host.load_1 orelse 0, host.load_5 orelse 0, host.load_15 orelse 0, + }); + const total_kib = host.mem_total_kib orelse 0; + const avail_kib = host.mem_available_kib orelse 0; + const used_gb = (total_kib -| avail_kib) / (1024 * 1024); + const total_gb = total_kib / (1024 * 1024); + const mem_str = try std.fmt.allocPrint(ctx.arena, "{d} / {d} GB", .{ used_gb, total_gb }); + const devices_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{self.state.deviceCount()}); + const values = [entry_count][]const u8{ + stamp.stamp("STABLE_ZML_SMI_VERSION"), + hostname_str, + kernel_str, + cpu_str, + cores_str, + mem_str, + uptime_str, + load_str, + devices_str, + }; + var entries: [entry_count]Entry = undefined; + for (&entries, labels, values) |*e, label, value| { + e.* = .{ .label = label, .value = value }; + } + + var sb = compose.surfaceBuilder(ctx.arena); + + for (entries, 0..) |entry, i| { + if (w == 0) { + continue; + } + + const total_len = entry.label.len + entry.value.len; + const segments = if (total_len < w) blk: { + const pad_len = w - total_len; + const pad = try ctx.arena.alloc(u8, pad_len); + @memset(pad, '-'); + pad[pad_len - 1] = ' '; + break :blk try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = pad, .style = theme.dim_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + } else try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + + const text_surf = try ui.drawRichLine(ctx, segments, w); + + try sb.add(@intCast(i), 0, text_surf); + } + + const total_h: u16 = @intCast(entries.len); + return sb.finish(.{ .width = w, .height = total_h }, ui.widget(self)); +} + +fn formatUptime(arena: std.mem.Allocator, seconds: u64) std.mem.Allocator.Error![]const u8 { + const days = seconds / 86400; + const hours = (seconds % 86400) / 3600; + const mins = (seconds % 3600) / 60; + if (days > 0) { + return std.fmt.allocPrint(arena, "{d}d {d}h {d}m", .{ days, hours, mins }); + } else if (hours > 0) { + return std.fmt.allocPrint(arena, "{d}h {d}m", .{ hours, mins }); + } else { + return std.fmt.allocPrint(arena, "{d}m", .{mins}); + } +} diff --git a/bin/zml-smi/tui/widgets/logo.zig b/bin/zml-smi/tui/widgets/logo.zig new file mode 100644 index 00000000..e6970485 --- /dev/null +++ b/bin/zml-smi/tui/widgets/logo.zig @@ -0,0 +1,63 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Image = @import("../lib/image.zig"); +const zml_logo = @import("zml/logo"); + +const Logo = @This(); + +style: vaxis.Cell.Style = .{}, +image: ?vaxis.Image = null, + +pub const logo_width: u16 = 28; +pub const logo_height: u16 = @intCast(zml_logo.zml_art_blocks[0].rows.len); + +pub fn draw(self: *const Logo, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + if (self.image) |img| { + const img_w: Image = .{ .image = img, .rows = logo_height }; + return img_w.draw(ctx); + } + + var sb = compose.surfaceBuilder(ctx.arena); + for (0..logo_height, 0..) |logo_row, row| { + try sb.add(@intCast(row), 0, try self.drawLogoRow(ctx, logo_row)); + } + return sb.finish(.{ .width = logo_width, .height = logo_height }, ui.widget(self)); +} + +fn drawLogoRow(self: *const Logo, ctx: vxfw.DrawContext, row_index: usize) std.mem.Allocator.Error!vxfw.Surface { + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + for (zml_logo.zml_art_blocks[0..]) |block| { + const row = block.rows[row_index]; + if (row.text.len == 0) continue; + + var view: std.unicode.Utf8View = .initUnchecked(row.text); + var iter = view.iterator(); + while (iter.nextCodepointSlice()) |codepoint| { + const shiny = zml_logo.isShineGlyph(codepoint); + try segments.append(ctx.arena, .{ + .text = codepoint, + .style = .{ + .fg = switch (if (shiny) row.shine else row.color) { + .reset => .default, + .cyan => .{ .rgb = .{ 0, 255, 255 } }, + .sky => .{ .rgb = .{ 0, 221, 255 } }, + .blue => .{ .rgb = .{ 135, 143, 255 } }, + .violet => .{ .rgb = .{ 215, 90, 219 } }, + .pink => .{ .rgb = .{ 242, 48, 174 } }, + .shine_cyan => .{ .rgb = .{ 140, 255, 255 } }, + .shine_sky => .{ .rgb = .{ 128, 244, 255 } }, + .shine_blue => .{ .rgb = .{ 205, 210, 255 } }, + .shine_violet => .{ .rgb = .{ 255, 170, 255 } }, + .shine_pink => .{ .rgb = .{ 255, 150, 220 } }, + }, + .bg = self.style.bg, + .bold = shiny, + }, + }); + } + } + return ui.drawRichLine(ctx, segments.items, logo_width); +} diff --git a/bin/zml-smi/tui/widgets/metric_card.zig b/bin/zml-smi/tui/widgets/metric_card.zig new file mode 100644 index 00000000..6ceb9069 --- /dev/null +++ b/bin/zml-smi/tui/widgets/metric_card.zig @@ -0,0 +1,165 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Gauge = @import("gauge.zig"); +const BrailleChart = @import("braille_chart.zig"); +const TitledBorder = @import("titled_border.zig"); + +const MetricCard = @This(); + +title: []const u8, +title_image: ?vaxis.Image = null, +title_right: ?[]const u8 = null, +cell_size: vxfw.Size = .{}, +/// Lines of "label: value" pairs to display. +lines: []const LineEntry = &.{}, +/// Optional gauge bars. +gauges: []const GaugeEntry = &.{}, +/// Optional braille mini-charts. +charts: []const ChartEntry = &.{}, +/// Whether this card is highlighted (e.g. hovered). +highlighted: bool = false, + +pub const LineEntry = struct { + label: []const u8, + value: []const u8, + style: vaxis.Cell.Style = theme.value_style, +}; + +pub const GaugeEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, +}; + +pub const ChartEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, + data: []const u8, // history values 0-100 + chart_height: u16 = 3, +}; + +pub fn draw(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const bstyle: vaxis.Cell.Style = if (self.highlighted) + .{ .fg = theme.accent, .bold = true } + else + theme.border_style; + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .title_image = self.title_image, + .value_label = self.title_right, + .cell_size = self.cell_size, + .border_style = bstyle, + }; + return tb.draw(ctx); +} + +fn maxFieldWidth(ctx: vxfw.DrawContext, entries: anytype, comptime field: []const u8) u16 { + var max_w: u16 = 0; + for (entries) |entry| { + const raw = @field(entry, field); + const value: []const u8 = if (@typeInfo(@TypeOf(raw)) == .optional) raw orelse continue else raw; + max_w = @max(max_w, @as(u16, @intCast(ctx.stringWidth(value)))); + } + return max_w; +} + +fn drawContent(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const inner_w = ctx.max.width orelse 40; + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 0; + + // Charts (gauge-like layout: label [braille] pct% suffix, braille spans multiple rows) + { + const max_chart_label_w = maxFieldWidth(ctx, self.charts, "label") + 1; // +1 trailing space + const max_chart_suffix_w = maxFieldWidth(ctx, self.charts, "suffix") + Gauge.sep_w; + + for (self.charts, 0..) |chart, ci| { + if (ci > 0) { + row += 1; + } + + const ch_h = chart.chart_height; + + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{chart.value}); + const pct_w: u16 = @intCast(ctx.stringWidth(pct_str)); + + const reserved = max_chart_label_w + pct_w + 1 + max_chart_suffix_w; + const braille_w: u16 = if (reserved < inner_w) inner_w - reserved else 0; + const text_row: i17 = row + @as(i17, @intCast(ch_h / 2)); + + // Label (vertically centered, col 0) + const label_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = chart.label, .style = theme.label_style }, + }, max_chart_label_w); + try sb.add(text_row, 0, label_surf); + + // Braille chart (all rows, after label) + if (braille_w > 0) { + const braille: BrailleChart = .{ .values = chart.data, .height = ch_h }; + const braille_surf = try braille.draw(ui.fixedSize(ctx, braille_w, ch_h)); + try sb.add(row, @intCast(max_chart_label_w), braille_surf); + } + + // Pct + suffix (vertically centered, after braille) + const right_col: i17 = @intCast(max_chart_label_w + braille_w); + const pct_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = " ", .style = .{} }, + .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(chart.value) } }, + }, pct_w + 1); + try sb.add(text_row, right_col, pct_surf); + + if (chart.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const suffix_col: i17 = @intCast(inner_w -| suffix_text_w); + const suffix_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = suffix, .style = theme.dim_style }, + }, suffix_text_w); + try sb.add(text_row, suffix_col, suffix_surf); + } + + row += @intCast(ch_h); + } + } + + // Gap between charts and gauges + if (self.charts.len > 0 and self.gauges.len > 0) { + row += 1; + } + + // Gauges (with 1-row gap between consecutive gauges) + const max_label_w = maxFieldWidth(ctx, self.gauges, "label") + 1; // +1 trailing space + const max_suffix_w = maxFieldWidth(ctx, self.gauges, "suffix") + Gauge.sep_w; + for (self.gauges, 0..) |ge, gi| { + const gauge: Gauge = .{ + .value = ge.value, + .label = ge.label, + .suffix = ge.suffix, + .label_reserve = max_label_w, + .suffix_reserve = max_suffix_w, + }; + const gauge_surf = try gauge.draw(ui.fixedSize(ctx, inner_w, 1)); + try sb.add(row, 0, gauge_surf); + row += 1; + if (gi + 1 < self.gauges.len) row += 1; // gap between gauges + } + + // Text lines + for (self.lines) |line| { + const text_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = line.label, .style = theme.label_style }, + .{ .text = line.value, .style = line.style }, + }, inner_w); + try sb.add(row, 0, text_surf); + row += 1; + } + + const content_h: u16 = @intCast(@max(row, 1)); + return sb.finish(.{ .width = inner_w, .height = content_h }, ui.drawWidget(self, drawContent)); +} diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig new file mode 100644 index 00000000..58b20d51 --- /dev/null +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -0,0 +1,183 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const TitledBorder = @import("titled_border.zig"); +const ColumnLayout = @import("column_layout.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessInfo = pi.ProcessInfo; + +const RichText = vxfw.RichText; +const Segment = vaxis.Cell.Segment; + +const ProcessTable = @This(); + +const max_visible_rows: u16 = 20; + +scroll_bars: vxfw.ScrollBars = .{ + .scroll_view = .{ + .children = .{ .builder = .{ .userdata = undefined, .buildFn = buildRow } }, + .draw_cursor = false, + }, + .draw_horizontal_scrollbar = false, +}, +merged: std.ArrayList(ProcessInfo) = .empty, + +pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { + self.merged.deinit(allocator); +} + +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { + self.scroll_bars.scroll_view.children.builder.userdata = self; + self.merged.clearRetainingCapacity(); + + for (state.process_lists) |pl| { + const pl_items = pl.front().items; + if (device_id) |did| { + for (pl_items) |entry| { + if (entry.device_idx != did) { + continue; + } + self.merged.append(state.gpa, entry) catch break; + } + } else { + self.merged.appendSlice(state.gpa, pl_items) catch break; + } + } + + // Enrich with host data (uid, username, cpu%, rss, cmdline) + state.enricher.enrich(state.io, self.merged.items); + + // Sort by dev_mem descending + std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { + fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { + return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); + } + }.cmp); + + const count: u32 = @intCast(self.merged.items.len); + self.scroll_bars.estimated_content_height = count; + self.scroll_bars.scroll_view.item_count = count; +} + +pub fn resetScroll(self: *ProcessTable) void { + self.scroll_bars.scroll_view.scroll = .{}; + self.scroll_bars.scroll_view.cursor = 0; +} + +pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const row_count = self.merged.items.len; + + var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; + const empty_rt: RichText = .{ + .text = &.{.{ .text = " No process data available", .style = theme.dim_style }}, + .softwrap = false, + .overflow = .clip, + }; + + var children: std.ArrayList(vxfw.Widget) = .empty; + try children.append(ctx.arena, header_rt.widget()); + + if (row_count == 0) { + try children.append(ctx.arena, try compose.sized( + ctx.arena, + try compose.pad(ctx.arena, empty_rt.widget(), .{ .top = 1 }), + .{ .width = w, .height = 2 }, + )); + } else { + const scroll_h: u16 = @min(@as(u16, @intCast(row_count)), max_visible_rows); + try children.append(ctx.arena, try compose.sized(ctx.arena, self.scroll_bars.widget(), .{ .width = w, .height = scroll_h })); + } + + const content: ColumnLayout = .{ .children = children.items }; + const tb: TitledBorder = .{ + .child = ui.widget(&content), + .title = "Processes", + }; + return tb.draw(ui.fixedWidth(ctx, w)); +} + +/// Builds only data rows (no header) for the scroll view. +fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { + const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); + if (idx >= self.merged.items.len) { + return null; + } + return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; +} + +// Column format strings — shared between header (comptime) and row (runtime). +const col_fmt = struct { + const pid = " {s: >7} "; + const user = "{s: <10} "; + const dev = "{s: <5} "; + const util = "{s: >6} "; + const mem = "{s: >8} "; + const cpu = "{s: >6} "; + const host_mem = "{s: >8} "; +}; + +const header_segs = blk: { + const s = theme.label_style; + break :blk [_]Segment{ + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.pid, .{"PID"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.user, .{"USER"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.dev, .{"DEV"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.util, .{"UTIL"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.mem, .{"DEV MEM"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.cpu, .{"CPU"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.host_mem, .{"HOST MEM"}) }, + .{ .style = s, .text = "Command" }, + }; +}; + +/// Format a value into a column: col(arena, col_fmt.pid, "{d}", .{pid}) +fn col(arena: std.mem.Allocator, comptime column: []const u8, comptime fmt: []const u8, args: anytype) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, column, .{try std.fmt.allocPrint(arena, fmt, args)}); +} + +fn drawProcessRow(ptr: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const info: *const ProcessInfo = @ptrCast(@alignCast(ptr)); + const val: vaxis.Style = .{ .fg = theme.text_primary }; + const dim: vaxis.Style = .{ .fg = theme.dim }; + + const segs = try ctx.arena.alloc(Segment, 8); + segs[0] = .{ .style = val, .text = try col(ctx.arena, col_fmt.pid, "{d}", .{info.pid}) }; + segs[1] = .{ .style = val, .text = try col(ctx.arena, col_fmt.user, "{s}", .{info.username[0..@min(info.username.len, 10)]}) }; + segs[2] = .{ .style = val, .text = try col(ctx.arena, col_fmt.dev, "{d}", .{info.device_idx}) }; + segs[3] = if (info.dev_util_percent) |util| + .{ .style = val, .text = try col(ctx.arena, col_fmt.util, "{d}%", .{util}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.util, "-", .{}) }; + segs[4] = if (info.dev_mem_kib) |mem| + .{ .style = val, .text = try col(ctx.arena, col_fmt.mem, "{s}", .{try fmtMem(ctx.arena, mem)}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.mem, "-", .{}) }; + segs[5] = .{ .style = val, .text = try col(ctx.arena, col_fmt.cpu, "{d}.{d}%", .{ info.cpu_percent / 10, info.cpu_percent % 10 }) }; + segs[6] = .{ .style = val, .text = try col(ctx.arena, col_fmt.host_mem, "{s}", .{try fmtMem(ctx.arena, info.rss_kib)}) }; + segs[7] = .{ .style = val, .text = info.comm }; + + const rich: RichText = .{ .text = segs, .softwrap = false, .overflow = .clip }; + return rich.draw(ctx.withConstraints( + .{ .width = ctx.min.width, .height = 1 }, + .{ .width = ctx.max.width, .height = 1 }, + )); +} + +fn fmtMem(arena: std.mem.Allocator, kib: u64) std.mem.Allocator.Error![]const u8 { + if (kib >= 1024 * 1024) { + const whole = kib / (1024 * 1024); + const frac = (kib % (1024 * 1024)) * 10 / (1024 * 1024); + return std.fmt.allocPrint(arena, "{d}.{d}G", .{ whole, frac }); + } else if (kib >= 1024) { + const whole = kib / 1024; + const frac = (kib % 1024) * 10 / 1024; + return std.fmt.allocPrint(arena, "{d}.{d}M", .{ whole, frac }); + } else { + return std.fmt.allocPrint(arena, "{d}K", .{kib}); + } +} diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig new file mode 100644 index 00000000..fb32541e --- /dev/null +++ b/bin/zml-smi/tui/widgets/status_line.zig @@ -0,0 +1,77 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); + +const StatusLine = @This(); + +viewing_device: ?u16, +use_braille: bool, + +const Binding = struct { + key: []const u8, + desc: []const u8, +}; + +pub fn draw(self: *const StatusLine, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + + const key_style: vaxis.Style = .{ .bold = true, .fg = .{ .rgb = .{ 30, 30, 40 } }, .bg = theme.accent }; + const desc_style: vaxis.Style = .{ .fg = theme.text_secondary }; + const sep_style: vaxis.Style = .{ .fg = theme.dim }; + const bg_style: vaxis.Style = .{ .bg = .{ .rgb = .{ 30, 30, 45 } } }; + + const chart_label = if (self.use_braille) "gauges" else "charts"; + + var bindings_buf: [8]Binding = undefined; + var n: usize = 0; + + if (self.viewing_device != null) { + bindings_buf[n] = .{ .key = "Esc", .desc = "back" }; + n += 1; + } else { + bindings_buf[n] = .{ .key = "v", .desc = chart_label }; + n += 1; + bindings_buf[n] = .{ .key = "click", .desc = "device details" }; + n += 1; + } + bindings_buf[n] = .{ .key = "q", .desc = "quit" }; + n += 1; + + const bindings = bindings_buf[0..n]; + + // Build segments: " key desc | key desc | ... " + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Leading space + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + + for (bindings, 0..) |b, i| { + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.key, .style = key_style }); + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.desc, .style = desc_style }); + if (i + 1 < bindings.len) { + try segments.append(ctx.arena, .{ .text = " \u{2502}", .style = sep_style }); + } + } + + const rich: vxfw.RichText = .{ + .text = segments.items, + .softwrap = false, + .overflow = .clip, + }; + const surf = try rich.draw(ui.fixedSize(ctx, w, 1)); + + // Fill background across full width + if (surf.buffer.len > 0) { + for (surf.buffer) |*cell| { + if (cell.style.bg == .default) { + cell.style.bg = bg_style.bg; + } + } + } + + return surf; +} diff --git a/bin/zml-smi/tui/widgets/titled_border.zig b/bin/zml-smi/tui/widgets/titled_border.zig new file mode 100644 index 00000000..d9c16171 --- /dev/null +++ b/bin/zml-smi/tui/widgets/titled_border.zig @@ -0,0 +1,112 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const Image = @import("../lib/image.zig"); + +const Text = vxfw.Text; + +const TitledBorder = @This(); + +child: vxfw.Widget, +title: []const u8 = "", +title_style: vaxis.Cell.Style = theme.title_style, +title_image: ?vaxis.Image = null, +cell_size: vxfw.Size = .{}, +value_label: ?[]const u8 = null, +value_label_style: vaxis.Cell.Style = theme.value_style, +border_style: vaxis.Cell.Style = theme.border_style, +padding: vxfw.Padding.PadValues = .{ .left = 1, .right = 1, .top = 1, .bottom = 1 }, + +pub fn draw(self: *const TitledBorder, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const pad: vxfw.Padding = .{ + .child = self.child, + .padding = self.padding, + }; + const border: vxfw.Border = .{ + .child = pad.widget(), + .style = self.border_style, + }; + var surf = try border.draw(ctx); + + // Count overlay children needed + var overlay_count: usize = 0; + var title_col: u16 = 2; + var has_image = false; + + if (self.title_image) |img| { + const icon = ui.imageCellSize(img, 1, self.cell_size); + has_image = true; + title_col += icon.cols; + overlay_count += 1; + } + if (self.title.len > 0) { + overlay_count += 1; + } + if (self.value_label != null) { + overlay_count += 1; + } + + if (overlay_count == 0) { + return surf; + } + + const new_children = try ctx.arena.alloc(vxfw.SubSurface, surf.children.len + overlay_count); + @memcpy(new_children[0..surf.children.len], surf.children); + var idx = surf.children.len; + + // Image icon overlay + if (has_image) { + const img = self.title_image.?; + const img_widget: Image = .{ .image = img, .rows = 1 }; + const img_surf = try img_widget.draw(ctx); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = 2 }, + .surface = img_surf, + .z_index = 1, + }; + idx += 1; + } + + // Title text overlay (top-left) + if (self.title.len > 0) { + const title_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{self.title}); + const title_widget: Text = .{ + .text = title_text, + .style = self.title_style, + .softwrap = false, + .overflow = .clip, + }; + const max_title_w = surf.size.width -| (title_col + 1); + const title_surf = try title_widget.draw(ui.maxSize(ctx, max_title_w, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(title_col) }, + .surface = title_surf, + .z_index = 1, + }; + idx += 1; + } + + // Value label overlay (top-right) + if (self.value_label) |vlabel| { + const label_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{vlabel}); + const label_widget: Text = .{ + .text = label_text, + .style = self.value_label_style, + .softwrap = false, + .overflow = .clip, + }; + const right = surf.size.width -| 1; + const lw: u16 = @intCast(@min(ctx.stringWidth(label_text), right -| 2)); + const label_surf = try label_widget.draw(ui.maxSize(ctx, lw, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(right -| lw) }, + .surface = label_surf, + .z_index = 1, + }; + } + + surf.children = new_children; + return surf; +} diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel new file mode 100644 index 00000000..a154556c --- /dev/null +++ b/bin/zml-smi/utils/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "double_buffer", + import_name = "zml-smi/double_buffer", + main = "double_buffer.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "dynlib", + import_name = "zml-smi/dynlib", + main = "dynlib.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "sandbox", + import_name = "zml-smi/sandbox", + main = "sandbox.zig", + visibility = ["//visibility:public"], + deps = ["@rules_zig//zig/runfiles"], +) + +zig_library( + name = "sysfs", + import_name = "zml-smi/sysfs", + main = "sysfs.zig", + visibility = ["//visibility:public"], +) diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig new file mode 100644 index 00000000..add3037b --- /dev/null +++ b/bin/zml-smi/utils/double_buffer.zig @@ -0,0 +1,32 @@ +const std = @import("std"); + +pub fn DoubleBuffer(comptime T: type) type { + return struct { + const Self = @This(); + pub const Value = T; + + values: [2]T, + current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), + + pub fn front(self: *const Self) *const T { + return &self.values[self.current.load(.acquire)]; + } + + pub fn back(self: *Self) *T { + return &self.values[1 - self.current.load(.acquire)]; + } + + pub fn swap(self: *Self) void { + self.current.store(1 - self.current.load(.acquire), .release); + } + + pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { + try jw.write(self.front().*); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { + const val = try std.json.innerParseFromValue(T, allocator, source, options); + return .{ .values = .{ val, val } }; + } + }; +} diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig new file mode 100644 index 00000000..4a478107 --- /dev/null +++ b/bin/zml-smi/utils/dynlib.zig @@ -0,0 +1,10 @@ +const std = @import("std"); + +pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { + var result: VTable = undefined; + inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { + @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; + } + + return result; +} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig new file mode 100644 index 00000000..d4e268bd --- /dev/null +++ b/bin/zml-smi/utils/sandbox.zig @@ -0,0 +1,44 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const bazel_runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +var runfiles_global: bazel_runfiles.Runfiles = undefined; + +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = argv; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + const io = threaded.io(); + + var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const exe_size = std.process.executablePath(io, &exe_path_buf) catch + std.debug.panic("Unable to get executable path", .{}); + + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = io, + .argv0 = exe_path_buf[0..exe_size], + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { + const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); + + return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; +} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig new file mode 100644 index 00000000..636883b2 --- /dev/null +++ b/bin/zml-smi/utils/sysfs.zig @@ -0,0 +1,70 @@ +const std = @import("std"); + +pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + const data = try readFirstLine(allocator, io, path); + return std.fmt.parseInt(u64, data, 10); +} + +pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + return readFirstLine(allocator, io, path); +} + +pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { + return nthTokenInt(try findField(allocator, io, path, key), 0); +} + +pub fn nthTokenInt(data: []const u8, n: usize) u64 { + var i: usize = 0; + + var iter = std.mem.tokenizeAny(u8, data, " \t"); + while (iter.next()) |tok| : (i += 1) { + if (i == n) { + return std.fmt.parseInt(u64, tok, 10) catch 0; + } + } + + return 0; +} + +pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + const value = try findField(allocator, io, path, key); + return std.mem.trimEnd(u8, value, &std.ascii.whitespace); +} + +fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var line_writer: std.Io.Writer.Allocating = .init(allocator); + errdefer line_writer.deinit(); + + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; + reader.interface.toss(1); + + const line = line_writer.written(); + if (std.mem.startsWith(u8, line, key)) { + const owned = line_writer.toOwnedSlice() catch return error.NotFound; + return std.mem.trimStart(u8, owned[key.len..], " :\t"); + } + } +} + +pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => 0, + else => |e| return e, + }; + return writer.toOwnedSlice(); +} diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 00000000..6c3be332 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,65 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1793199252, + "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", + "owner": "cachix", + "repo": "devenv", + "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "nixpkgs": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1793043970, + "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1792812153, + "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 00000000..cafd0f90 --- /dev/null +++ b/devenv.nix @@ -0,0 +1,20 @@ +{ + pkgs, + config, + ... +}: { + packages = with pkgs; [ + bazelisk + python3 + uv + zig + ]; + env.ZLS_CMD = "${config.devenv.root}/tools/zls.sh"; + scripts.bazel.exec = ''bazelisk "$@"''; + + enterShell = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + # Use the "real" XCode rather than Nixpkg's one. This does assume you have it installed. + export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer" + export SDKROOT="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"; + ''; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 00000000..116a2adb --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json +inputs: + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + +# If you're using non-OSS software, you can set allowUnfree to true. +# allowUnfree: true + +# If you're willing to use a package that's vulnerable +# permittedInsecurePackages: +# - "openssl-1.1.1w" + +# If you have more than one devenv you can merge them +#imports: +# - ./backend diff --git a/docs/README.md b/docs/README.md index eedfdd9b..03369120 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,9 +17,9 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## How to ? - [HuggingFace Authentication](./howtos/huggingface_access_token.md) - [Port Pytorch models to ZML](./howtos/howto_torch2zml.md) -- [Add Weights Files to your projects](./howtos/add_weights.md) - [Cross Compile and Deploy Models on a Server](./howtos/deploy_on_server.md) - [Dockerize Models](./howtos/dockerize_models.md) +- [Profile ZML Programs](./howtos/profiling.md) ## Learn more... @@ -28,4 +28,3 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Contribute - [Style Guide](./misc/style_guide.md) - diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md deleted file mode 100644 index 2caf2fc9..00000000 --- a/docs/howtos/add_weights.md +++ /dev/null @@ -1,69 +0,0 @@ - -# Adding Weights Files - -Our [first model](../tutorials/write_first_model.md) did not need any weights files. -We just created weights and biases at runtime. - -But real-world models typically need weights files, and maybe some other -supporting files. - -We recommend, for easy deployments, you upload those files. In many instances, -you will use a site like [🤗 Hugging Face](https://huggingface.co). - -We also recommend to add a `weights.bzl` file to your project root directory, so -you don't "pollute" your build file with long URLs and SHAs: - -```python -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _weights_impl(mctx): - http_archive( - name = "mnist", - sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", - url = "https://mirror.zml.ai/data/mnist.tar.zst", - build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -weights = module_extension( - implementation = _weights_impl, -) -``` - -The above `weights.bzl` shows how we load files for MNIST: - -- `mnist.pt` (model weights) -- `mnist.ylc` (dataset for picking sample images) - -Then, in your `BUILD.bazel`, you can refer to the files you defined above, in -the following way: - -```python -zig_binary( - name = "mnist", - args = [ - "$(location @mnist//:mnist.pt)", - "$(location @mnist//:t10k-images.idx3-ubyte)", - ], - data = [ - "@mnist//:mnist.pt", - "@mnist//:t10k-images.idx3-ubyte", - ], - main = "mnist.zig", - deps = [ - "//async", - "//zml", - ], -) -``` - -See how: - -- we use `data = [ ... ]` to reference the files in `weights.bzl` -- we use `args = [ ... ]` to pass the files as command-line arguments to the - MNIST executable at runtime, automatically. diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index 05a88889..a8a6452e 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -24,7 +24,11 @@ So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama --@zml//platforms:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --prompt="What is the capital of France?" ``` @@ -40,7 +44,7 @@ architectures: As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` -bazel build --config=release //examples/llama \ +bazel build --config=release //examples/llm \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 @@ -57,7 +61,8 @@ If not present already, add an "archive" target to the model's `BUILD.bazel`, like this: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # Manifest, required for building the tar archive mtree_spec( @@ -81,15 +86,15 @@ tar( ... and then build the TAR archive: ``` -bazel build --config=release //mnist:archive \ +bazel build --config=release //examples/mnist:archive \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` -Note the `//mnist:archive` notation. +Note the `//examples/mnist:archive` notation. -The resulting tar file will be in `bazel-bin/mnist/archive.tar.zst`. +The resulting tar file will be in `bazel-bin/examples/mnist/archive.tar.zst`. ### Run it on the server @@ -98,36 +103,35 @@ and run it: ```bash # on your machine -scp bazel-bin/mnist/archive.tar.zst destination-server: +scp bazel-bin/examples/mnist/archive.tar.zst destination-server: ssh destination-server # to enter the server # ... on the server tar xvf archive.tar.zst -./mnist \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt' \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc' +./examples/mnist/mnist \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/mnist.safetensors \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/t10k-images.idx3-ubyte ``` The easiest way to figure out the commandline arguments of an example model is to consult the model's `BUILD.bazel` and check out its `args` section. It will -reference e.g. weights files that are defined either in the same `BUILD.bazel` -file or in a `weights.bzl` file. +reference e.g. weights files that are defined in the same `BUILD.bazel`. You can also consult the console output when running your model locally: ```bash -bazel run //mnist +bazel run //examples/mnist -INFO: Analyzed target //mnist:mnist (0 packages loaded, 0 targets configured). +INFO: Analyzed target //examples/mnist:mnist (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //mnist:mnist up-to-date: - bazel-bin/mnist/mnist -INFO: Elapsed time: 0.302s, Critical Path: 0.00s -INFO: 3 processes: 3 internal. -INFO: Build completed successfully, 3 total actions -INFO: Running command line: bazel-bin/mnist/mnist ../_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt ../_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc -# ... +Target //examples/mnist:mnist up-to-date: + bazel-bin/examples/mnist/mnist +INFO: Elapsed time: 0.114s, Critical Path: 0.00s +INFO: 1 process: 1 internal. +INFO: Build completed successfully, 1 total action +INFO: Running command line: bazel-bin/examples/mnist/mnist ../+non_module_deps+mnist/mnist.safetensors ../+non_module_deps+mnist/t10k-images.idx3-ubyte. +# .. ``` You see the command line right up there. On the server, you just need to replace -`../` with the 'runfiles' directory of your TAR. +`../` with the absolute path to the file. diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index ac35990c..7414b985 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -38,16 +38,16 @@ We need to add a few "imports" at the beginning of our `BUILD.bazel` so we can use their rules to define our 5 additional targets: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -135,11 +135,11 @@ And that's almost it! You can already build the image: ``` bazel build --config=release //examples/simple_layer:image -INFO: Analyzed target //simple_layer:image (1 packages loaded, 8 targets configured). +INFO: Analyzed target //examples/simple_layer:image (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:image up-to-date: - bazel-out/k8-dbg-ST-f832ad0148ae/bin/simple_layer/image_ -INFO: Elapsed time: 0.279s, Critical Path: 0.00s +Target //examples/simple_layer:image up-to-date: + bazel-out/linux_amd64-opt/bin/examples/simple_layer/image_ +INFO: Elapsed time: 0.285s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action ``` @@ -168,7 +168,7 @@ oci_load( ... then we can load the image and run it with the following commands: ``` -bazel run --config=release //simple_layer:load +bazel run --config=release //examples/simple_layer:load docker run --rm distroless/simple_layer:latest ``` @@ -193,7 +193,7 @@ This will push the `simple_layer` image with the tag `latest` (you can add more) to the docker registry: ``` -bazel run --config=release //simple_layer:push +bazel run --config=release //examples/simple_layer:push ``` When dealing with maybe a public and a private container registry - or if you @@ -201,7 +201,7 @@ just want to try it out **right now**, you can always override the repository on the command line: ``` -bazel run --config=release //simple_layer:push -- --repository my.server.com/org/image +bazel run --config=release //examples/simple_layer:push -- --repository my.server.com/org/image ``` @@ -215,8 +215,8 @@ We'll use the [MNIST example](https://github.com/zml/zml/tree/master/examples/mnist) to illustrate how to build Docker images that also contain data files. -You can `bazel run --config=release //mnist:push -- --repository -index.docker.io/my_org/zml_mnist` in the `./examples` folder if you want to try +You can `bazel run --config=release //examples/mnist:push -- --repository +index.docker.io/my_org/zml_mnist` if you want to try it out. **Note: Please add one more of the following parameters to specify all the @@ -231,7 +231,7 @@ platforms your containerized model should support.** **Example:** ``` -bazel run //mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` @@ -244,26 +244,27 @@ same. Let's start with creating the manifest and archive: ```python -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # The executable zig_binary( name = "mnist", args = [ - "$(location @com_github_ggerganov_ggml_mnist//file)", - "$(location @com_github_ggerganov_ggml_mnist_data//file)", + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", + visibility = ["//visibility:public"], deps = [ - "@zml//async", + "@zml//bazel", "@zml//zml", ], ) @@ -296,22 +297,24 @@ simple string interpolation will not be enough. For this reason, we use the `expand_template` rule, like this: ```python +load("@bazel_lib//lib:expand_template.bzl", "expand_template") + # A convenience template for creating the "command line" for the entrypoint expand_template( name = "entrypoint", data = [ ":mnist", - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], substitutions = { - ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", - ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", }, template = [ "./{}/mnist".format(package_name()), - "./{}/mnist.runfiles/:model".format(package_name()), - "./{}/mnist.runfiles/:data".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), ], ) ``` @@ -372,5 +375,5 @@ And that's it! With one simple bazel command, you can push a neatly packaged MNIST model, including weights and dataset, to the docker registry: ``` -bazel run //mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 5c751c53..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -67,14 +67,15 @@ package, so the simplest way to use it, is to copy it next to your python script. Then wrap the model/pipeline in a `zml_utils.ActivationCollector`. The collector wraps the given model, and returns the original results AND the activations in a dict of `torch.Tensor` when it's being called. After that, you -can save those activations to a `.pt` file. +can save those activations to a `.safetensors` file. ```python import torch import transformers +from safetensors.torch import save_file import zml_utils -model_path = "meta-llama/Meta-Llama-3-8B" +model_path = "meta-llama/Llama-3.2-1B-Instruct" pipeline = transformers.pipeline( "text-generation", @@ -97,8 +98,8 @@ if output: print(output) # Save activations to a file. -filename = model_path.split("/")[-1] + ".activations.pt" -torch.save(activations, filename) +filename = model_path.split("/")[-1] + ".activations.safetensors" +save_file(activations, filename) print(f"Saved {len(activations)} activations to {filename}") ``` @@ -118,36 +119,27 @@ model. Put the following in `my_project/torch2zml.zig`. const std = @import("std"); const log = std.log; -const async = @import("async"); const zml = @import("zml"); -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} - -pub fn asyncMain() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = try init.minimal.args.toSlice(allocator); + defer allocator.free(args); const model_path, const activations_path = args[1..3].*; - const activations = try zml.aio.torch.open(allocator, activations_path); - defer activations.deinit(); - log.info("Found {} activations in {s}", .{ activations.buffers.count(), activations_path }); + var activations_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, activations_path); + defer activations_registry.deinit(); + log.info("Found {} activations in {s}", .{ activations_registry.tensors.count(), activations_path }); - const model_weights = try zml.aio.detectFormatAndOpen(allocator, model_path); - defer model_weights.deinit(); - log.info("Found {} model layers in {s}", .{ model_weights.buffers.count(), activations_path }); + var model_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, model_path); + defer model_registry.deinit(); + log.info("Found {} activations in {s}", .{ model_registry.tensors.count(), model_path }); } ``` -And add a `zig_cc_binary` target in `my_project/BUILD.bazel`: +And add a `zig_binary` target in `my_project/BUILD.bazel`: ```python load("@rules_zig//zig:defs.bzl", "zig_binary") @@ -156,7 +148,6 @@ zig_binary( name = "torch2zml", main = "torch2zml.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -166,19 +157,16 @@ Now check that the weights can be loaded correctly using the bazel CLI. ```bash bazel build //my_project:torch2zml -./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Meta-Llama-3-8B.activations.pt +./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors -info: Found 1108 activations in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt -debug(zml_io): Loading shard: model-00004-of-00004.safetensors -debug(zml_io): Loading shard: model-00001-of-00004.safetensors -debug(zml_io): Loading shard: model-00002-of-00004.safetensors -debug(zml_io): Loading shard: model-00003-of-00004.safetensors -info: Found 291 model layers in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt +info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors +debug(zml/safetensors): Added new metadata key=format with value=pt +info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` -## Loading an individual layer +## Initializing an individual layer -In the above Zig code, the `model_weights` struct is a wrapper around a flat +In the above Zig code, the `model_registry` struct is a wrapper around a flat dictionary, containing an entry for each tensor in the model (similar to a "state dict"). Manipulating a dictionary is generally not very convenient, so let's convert it to a Zig struct. @@ -196,26 +184,70 @@ const Mlp = struct { The `zml.nn.Linear` is the equivalent of `torch.nn.Linear` and is defined by its `weight` and optional `bias` tensors. -To create such a struct from our `model_weights` dictionary, we can use the -`zml.aio.populateModelWithPrefix` helper: +To create such a struct from our `model_registry` dictionary, we first need to +convert it to a `TensorStore`: ```zig -pub fn asyncMain() !void { +pub fn main(init: std.process.Init) !void { ... - const mlp_shape = try zml.aio.populateModelWithPrefix(Mlp, allocator, model_weights, "model.layers.0.mlp"); - log.info("layer.0.mlp: {}", .{mlp_shape}); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); + defer model_store.deinit(); +} +``` + +Then, we can initialize our MLP: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + const mlp_view = model_store.view().withPrefix("model.layers.0.mlp"); + + const mlp: Mlp = .{ + .up_proj = .init( + mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), + .d, + ), + .gate_proj = .init( + mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), + .d, + ), + .down_proj = .init( + mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), + .dout, + ), + }; } ``` Build and run, using previous commands. -Typical errors are of the form _"Layer not found: ..."_. This is typically due -to the naming of layers in Zig not matching the naming in the file. +Typical errors are due to the naming of layers in Zig not matching the naming +in the file. Double-check everything and don't hesitate to print more things, e.g. in the Python script. Alternatively, Huggingface's web-interface allows to peek into `.safetensor` files. +## Loading an individual layer + +Currently, the layer is initialized but the weights are still on the disk. +ZML provides a convenience to automatically load weights from an initialized model: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); +} +``` + + ## Testing an individual layer Finally, we are going to write the actual math code for our `MLP` layer. @@ -226,33 +258,39 @@ const Mlp = struct { gate_proj: zml.nn.Linear, down_proj: zml.nn.Linear, - pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + return self.down_proj.forward(output, x.dtype()); } }; ``` -Note that we use `zml.call` instead of directly calling -`self.up_proj.forward(x)`. Calling `forward` directly results in the same -computation happening at runtime; but going through `zml.call` allows ZML to -generate an MLIR representation that is closer to the Zig code and therefore -easier to read. - -We can test the MLP layer with the `zml.testing.testLayer` utility: +Our MLP layer is now almost ready to be tested with the `zml.testing.testLayer` utility. +As you may have noticed, ZML makes heavy-use of tagged tensors. In the MLP, we expect +to do a dot product on the `.d` dimension. But the activations we load from the +activations files are not tagged. One solution could be to tag the tensors before saving +them to file, but a much simpler solution is to intercept the inputs before providing +them to the layer we want to test, using a wrapper. ```zig -pub fn asyncMain() !void { +pub const Wrapper = struct { + mlp: Mlp, + + pub fn forward(wrapper: Wrapper, input: zml.Tensor) zml.Tensor { + const tagged_input = input.withTags(.{.b, .s, .d}); + return wrapper.mlp.forward(tagged_input); + } +}; + +pub fn main(init: std.process.Init) !void { ... - var ctx = try zml.Context.init(); - defer ctx.deinit(); - const platform = ctx.autoPlatform(.{}); - const mlp_weights = try zml.aio.loadModelBuffers(Mlp, mlp_shape, model_weights, allocator, platform); + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); + defer activations_store.deinit(); - zml.testing.testLayer(platform, activations, "model.layers.0.mlp", mlp_shape, mlp_weights, 1e-3); + try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); } ``` diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index 797c6dfb..cc11ffbb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -3,7 +3,7 @@ Some models have restrictions and may require some sort of approval or agreement process, which, by consequence, **requires token-authentication with Huggingface**. -The easiest way might be to use the `huggingface-cli login` command. +The easiest way might be to use the `hf auth login` command. Alternatively, here is how you can generate a **"read-only public repositories"** access token to log into your account on Huggingface, directly from `bazel`, in order to download models. @@ -29,7 +29,6 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/howtos/profiling.md b/docs/howtos/profiling.md new file mode 100644 index 00000000..f98a7dc2 --- /dev/null +++ b/docs/howtos/profiling.md @@ -0,0 +1,375 @@ +# Profiling ZML Programs + +ZML has two profiling layers: + +- host tracing through `zml.tracer.scope(...)` +- backend profiling through `zml.Platform.profiler(...)` + +The host tracing path is always based on `TraceMe`. Depending on the target +platform, the tracing bridge also emits device-visible annotations: + +- CUDA on Linux: NVTX ranges +- ROCm on Linux: ROCTx ranges +- macOS: `os_signpost` intervals for Instruments + +When backend profiling is enabled, ZML writes both the raw `XSpace` protobuf +and a streamed Perfetto / trace-viewer JSON file. + +## Profiling architecture + +ZML profiling is split into four layers: + +1. `zml.tracer.scope(...)` in Zig marks lexical regions of interest +2. `zml/profiling/traceme_lib.cc` bridges those scopes into platform-specific + tracing backends +3. `zml.Platform.profiler(...)` manages a profiling session and collects profile + data from PJRT +4. `tools/xspace_to_perfetto` converts the final `XSpace` protobuf into a + streamed trace-event JSON file + +The important architectural split is: + +- `zml.tracer.scope(...)` is for instrumentation inside your code +- `zml.Platform.profiler(...)` is for session lifecycle and artifact generation + +You can use scopes without a profiler session, but you only get exported +`profiling.xplane.pb` and `profiling.trace.json` when a profiler session is +started. + +### Scope instrumentation path + +`zml.tracer.scope(...)` is the lowest-level API you use directly in Zig code. +Each scope creates a `TraceMe` region, and the C bridge adds extra +platform-visible annotations when supported: + +- CUDA/Linux: NVTX ranges +- ROCm/Linux: ROCTx ranges +- macOS: `os_signpost` intervals + +Metadata passed to `zml.tracer.scope(...)` is encoded into the `TraceMe` scope +name. That is why the API only accepts small scalar or string-like fields: it is +designed to produce compact trace labels rather than structured protobuf +payloads. + +### Profiler session path + +`zml.Platform.profiler(...)` builds a `tensorflow.ProfileOptions` protobuf from +`zml.Platform.ProfilerOptions` and asks the active PJRT backend for a profiler +session. + +At `start()` time, ZML does two distinct things: + +- it tries to start the backend PJRT profiler session +- it configures local host-side `TraceMeRecorder` capture when enabled + +At `stop()` time, ZML: + +1. stops the PJRT profiler session if one exists +2. collects the backend `XSpace` protobuf +3. merges locally recorded host `TraceMe` events into the host plane +4. writes the final protobuf to disk +5. streams trace JSON from that final protobuf + +This separation is important because host scopes and backend profiler output are +not produced by the same subsystem. ZML joins them only at the end of the +session. + +### Export path + +The final export path is intentionally one-way: + +- source data format: `XSpace` protobuf +- derived export format: trace-event JSON for Perfetto / trace viewer + +ZML keeps `profiling.xplane.pb` as the source of truth and generates +`profiling.trace.json` from it afterwards. The converter is implemented in Zig +and streams JSON directly from the protobuf instead of materializing the whole +JSON document in memory first. + +## The generated artifacts + +By default, a profiling session is written under: + +```text +/tmp/xprof/plugins/profile/profiling/ +``` + +The relevant files are: + +- `profiling.xplane.pb`: raw `XSpace` protobuf +- `profiling.trace.json`: Perfetto / trace-viewer JSON export + +These paths come from `zml/profiling/profiler.zig` and can be changed through +`zml.Platform.ProfilerOptions`. + +## Add trace scopes in Zig + +Use `zml.tracer.scope(name, metadata)` to bracket the code you want to inspect. + +```zig +const zml = @import("zml"); + +var trace = try zml.tracer.scope("llm.compile", .{ + .seqlen = args.seqlen, + .interactive = args.prompt == null, +}); +defer trace.end(); + +try models.LoadedModel.compile(...); +``` + +Use `.{}` when there is no metadata: + +```zig +var trace = try zml.tracer.scope("llm.load_buffers", .{}); +defer trace.end(); +``` + +Metadata is encoded into the `TraceMe` scope name, so keep it small and +high-signal. + +## Record a ZML profile from code + +If you want ZML to collect a backend profile and export `profiling.trace.json`, +create a profiler from the current platform: + +```zig +const platform: *zml.Platform = try .auto(allocator, io, .{}); + +var profiler = try platform.profiler(allocator, io, .defaults); +defer profiler.deinit(); + +try profiler.start(); +defer { + if ((profiler.stop() catch unreachable)) |profile| { + std.log.info("Profile dumped: {s} and {s}", .{ + profile.protobuf_path, + profile.perfetto_path, + }); + } +} +``` + +This is the normal ZML flow: + +1. start the PJRT backend profiler +2. record local `TraceMe` scopes +3. merge the local host trace into the backend `XSpace` +4. write `profiling.xplane.pb` +5. stream `profiling.trace.json` + +If the backend profiler is unavailable, ZML still keeps the local host trace +path and logs a warning. + +## CUDA on Linux with Nsight Systems + +ZML provides a Bazel run config for `nsys`: + +```bash +bazel run --config=nsys //examples/benchmark:benchmark +``` + +Or for a real model: + +```bash +bazel run --config=nsys //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:nsys`. By default it runs: + +```text +sudo -E nsys profile ... +``` + +with CUDA, cuBLAS, cuSPARSE, cuDNN, NVTX, syscall, and OS runtime tracing +enabled. + +If you need to install Nsight Systems manually, use NVIDIA's official download +page: . For the server +or target machine where `nsys` will run, choose the `Linux CLI only` package. +For the client machine where you open and inspect reports, choose the full host +package for your OS. NVIDIA's installation guide is here: +. + +Useful overrides: + +- `ZML_PROFILE_NO_SUDO=1`: run `nsys` without `sudo` +- `ZML_NSYS_ARGS='...'`: replace the default `nsys` arguments + +Example: + +```bash +ZML_PROFILE_NO_SUDO=1 bazel run --config=nsys //examples/benchmark:benchmark +``` + +### Important interaction with `zml.profiler` + +The `nsys` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That tells `zml/profiling/profiler.zig` to skip nested PJRT backend profiling. This is +intentional. Under `nsys`, you normally want: + +- the external Nsight session +- local `TraceMe` / NVTX annotations from your scopes + +and not a second backend profile session layered on top. + +## ROCm on Linux with `rocprofv3` + +ZML also provides a Bazel run config for ROCm: + +```bash +bazel run --config=rocprofv3 //examples/benchmark:benchmark +``` + +Or: + +```bash +bazel run --config=rocprofv3 //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:rocprofv3`. + +This path is intentionally sandboxed inside the repository's ROCm packaging: + +- it launches the Bazel-runfiles copy of `rocprofv3` +- it passes `--rocm-root` pointing at the sandboxed ROCm runtime + +So it does not depend on a host-installed `rocprofv3` layout. + +The default wrapper enables system, runtime, HIP, kernel, memory copy, +scratch-memory, HSA core, and marker tracing, and writes `pftrace` output. + +Useful overrides: + +- `ZML_ROCPROFV3_ARGS='...'`: replace the default `rocprofv3` arguments + +### Important interaction with `zml.profiler` + +Like the `nsys` wrapper, the `rocprofv3` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That means: + +- the external `rocprofv3` session owns backend/device profiling +- ZML still records local host `TraceMe` scopes +- ROCTx ranges from `zml.tracer.scope(...)` stay visible as device annotations + +## Neuron Profiling + +Use `--config=neuron-profile` with the command you already run. The wrapper +prints the profile directory when the program exits. + +```bash +NEURON_RT_VISIBLE_CORES=0,1 \ +bazel run --config=neuron-profile //examples/llm -- \ + --model=/path/to/model \ + --prompt="Explain mixture-of-experts routing in one paragraph." \ + --seqlen=128 \ + --backend=nki \ + --topk=1 +``` + +The default profile root is `/tmp/zml-neuron-profile`. Each run contains: + +```text +/tmp/zml-neuron-profile// + compile/ + execution/ +``` + +Start the Neuron Explorer server in one terminal: + +```bash +bazel run //tools/neuron:server +``` + +Open . When working through SSH, forward ports `3001` +and `3002`. + +With the server running, ingest the run from another terminal: + +```bash +bazel run //tools/neuron:ingest -- /tmp/zml-neuron-profile/ +``` + +Create a quick summary from one run when you want a file artifact: + +```bash +bazel run //tools/neuron:summary-json -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-txt -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-perfetto -- /tmp/zml-neuron-profile/ +``` + +Each summary command writes beside the run and prints the output path: +`summary.json`, `summary.txt`, or `summary.pftrace`. + +## macOS Instruments + +On macOS, `zml.tracer.scope(...)` emits `os_signpost` intervals. These show up +in Instruments, especially in the `Points of Interest` template. + +Build the binary first: + +```bash +bazel build //examples/llm:llm +``` + +Then either launch it from the Instruments UI or use `xctrace`: + +```bash +xctrace record \ + --template 'Points of Interest' \ + --output /tmp/llm.trace \ + --launch -- \ + /home/you/zml/bazel-bin/examples/llm/llm \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +Use full Xcode for `xctrace`. Command Line Tools alone are not enough. + +## Open the exported trace + +The `profiling.trace.json` file can be opened in Perfetto or in tools that still +understand the trace-event JSON format. + +Typical flow: + +1. run a ZML program with `zml.Platform.profiler(...)` +2. wait for `profiling.trace.json` +3. open the JSON in Perfetto + +The converter is implemented in Zig and streams JSON directly from `XSpace`, +which keeps large traces much more manageable than the old in-memory path. + +## Platform summary + +- Plain ZML profiling: backend `XSpace` profile plus merged local host trace +- CUDA + `--config=nsys`: external Nsight session plus local `TraceMe` / NVTX +- ROCm + `--config=rocprofv3`: external `rocprofv3` session plus local `TraceMe` / ROCTx +- macOS Instruments: local `TraceMe` scopes exposed as `os_signpost` + +## Troubleshooting + +- If you run under `--config=nsys`, `--config=rocprofv3`, or + `--config=neuron-profile`, PJRT profiling is skipped on purpose because the + wrappers set `SKIP_PJRT_PROFILER=true`. +- If you do not see device annotations, first check whether you are on a + supported platform: + - CUDA/Linux for NVTX + - ROCm/Linux for ROCTx + - macOS for Instruments signposts +- If you only need host scopes, `zml.tracer.scope(...)` works independently of + whether backend profiling succeeds. diff --git a/docs/learn/concepts.md b/docs/learn/concepts.md index de14e15b..1a3cea05 100644 --- a/docs/learn/concepts.md +++ b/docs/learn/concepts.md @@ -8,8 +8,8 @@ particulary Neural Networks (NN). The lifecycle of a model is implemented in the following steps: -1. Open the model file and read the shapes of the weights, but leave the - weights on the disk. +1. Open the model file and read the shapes of the weights, but don't load the weights + yet. 2. Using the loaded shapes and optional metadata, instantiate a model struct with `Tensor`s, representing the shape and layout of each layer of the NN. @@ -20,28 +20,21 @@ The lifecycle of a model is implemented in the following steps: 4. Load the model weights from disk, onto the accelerator memory. -5. Bind the model weights to the executable. +5. Load some user inputs, and copy them to the accelerator. -6. Load some user inputs, and copy them to the accelerator. +6. Call the executable using the weights and the user inputs. -7. Call the executable on the user inputs. - -8. Fetch the returned model output from accelerator into host memory, and +7. Fetch the returned model output from accelerator into host memory, and finally present it to the user. -9. When all user inputs have been processed, free the executable resources and - the associated weights. +8. When all user inputs have been processed, free the executable resources, weights and + inputs. **Some details:** Note that the compilation and weight loading steps are both bottlenecks to your -model startup time, but they can be done in parallel. **ZML provides -asynchronous primitives** to make that easy. - -The **compilation can be cached** across runs, and if you're always using the -same model architecture with the same shapes, it's possible to by-pass it -entirely. +model startup time, but they can be done in parallel thanks to Zig's `std.Io` interface. The accelerator is typically a GPU, but can be another chip, or even the CPU itself, churning vector instructions. @@ -51,7 +44,7 @@ itself, churning vector instructions. In ZML, we leverage Zig's static type system to differentiate between a few concepts, hence we not only have a `Tensor` to work with, like other ML -frameworks, but also `Buffer`, `HostBuffer`, and `Shape`. +frameworks, but also `Buffer`, `Slice`, and `Shape`. Let's explain all that. @@ -65,25 +58,19 @@ Let's explain all that. `Shape` struct can also represent a regular number, aka a scalar: `Shape.init(.{}, .i32)` represents a 32-bit signed integer. -* `HostBuffer`: _is_ a multi-dimensional array, whose memory is allocated **on - the CPU**. - - points to the slice of memory containing the array - - typically owns the underlying memory - but has a flag to remember when it - doesn't. +* `Slice`: _is_ the combination of a `Shape` and raw bytes (bytes that are **on the CPU**). + - can own the underlying memory - but can also accomodate non-owned memory. * `Buffer`: _is_ a multi-dimension array, whose memory is allocated **on an accelerator**. - contains a handle that the ZML runtime can use to convert it into a physical address, but there is no guarantee this address is visible from the CPU. - - can be created by loading weights from disk directly to the device via - `zml.aio.loadBuffers` - - can be created by calling `HostBuffer.toDevice(accelerator)`. + - can be created from a `Slice` by calling `Buffer.fromSlice(...)`. * `Tensor`: is a mathematical object representing an intermediary result of a - computation. - - is basically a `Shape` with an attached MLIR value representing the - mathematical operation that produced this `Tensor`. + computation or an input to an executable (including weights). + - is basically a `Shape` with an attached MLIR value when in the context of a compilation. ## The model struct @@ -137,25 +124,22 @@ struct of `Model` but with a `Buffer` replacing every `Tensor`. Let's look at the model life cycle again, but this time annotated with the corresponding types. -1. Open the model file and read the shapes of the weights -> `zml.HostBuffer` - (using memory mapping, no actual copies happen yet) +1. Open the model file and read the shapes of the weights -> `zml.Shape` 2. Instantiate a model struct -> `Model` struct (with `zml.Tensor` inside) 3. Compile the model struct and its `forward` function into an executable. `foward` is a `Tensor -> Tensor` function, executable is a - `zml.FnExe(Model.forward)` + `zml.Executable` 4. Load the model weights from disk, onto accelerator memory -> `zml.Bufferized(Model)` struct (with `zml.Buffer` inside) -5. Bind the model weights to the executable `zml.ModuleExe(Model.forward)` +5. Load some user inputs (custom struct), encode them into arrays of numbers + (`zml.Slice`), and copy them to the accelerator (`zml.Buffer`). -6. Load some user inputs (custom struct), encode them into arrays of numbers - (`zml.HostBuffer`), and copy them to the accelerator (`zml.Buffer`). - -7. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` +6. Call the executable on the user inputs. `exe.call` accepts `zml.Buffer` arguments and returns `zml.Buffer` -8. Return the model output (`zml.Buffer`) to the host (`zml.HostBuffer`), +7. Return the model output (`zml.Buffer`) to the host (`zml.Slice`), decode it (custom struct) and finally return to the user. diff --git a/docs/misc/style_guide.md b/docs/misc/style_guide.md index 165555ce..a112377c 100644 --- a/docs/misc/style_guide.md +++ b/docs/misc/style_guide.md @@ -1,7 +1,7 @@ # ZML Style Guide -We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.13.0/#Style-Guide). +We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.15.1/#Style-Guide). We use ZLS to auto-format code. diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index 31f8b4db..ababeb15 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -21,14 +21,14 @@ through `bazelisk`, a version manager for `bazel`. **macOs:** ``` - brew install bazelisk +brew install bazelisk ``` **Linux:** ``` - curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' - chmod +x /usr/local/bin/bazel +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/bazelisk-linux-amd64' +chmod +x /usr/local/bin/bazel ``` @@ -49,7 +49,7 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -bazel run --config=release //mnist +bazel run --config=release //examples/mnist ``` ### Llama @@ -71,11 +71,10 @@ Once you've been granted access, you're ready to download a gated model like `Meta-Llama-3.1-8B-Instruct`! ``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +# Authenticate on Huggingface. +bazel run //tools/hf -- auth login +# Run the model, loading directly from HF. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -86,9 +85,8 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' -bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +bazel run //tools/hf -- auth login +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. @@ -114,13 +112,15 @@ following arguments to the command line when compiling or running a model: The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, +So, to run the Llama model from above on your host sporting an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llama:Llama-3.2-1B-Instruct \ - --@zml//platforms:cuda=true \ - -- --prompt="What is the capital of France?" +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.1-8B-Instruct \ + --prompt="What is the capital of France?" ``` diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index ebaf6c2f..312c9afc 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -42,13 +42,6 @@ touch simple_layer/main.zig touch simple_layer/BUILD.bazel ``` -By the way, you can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - - - ## The high-level Overview Before firing up our editor, let's quickly talk about a few basic ZML @@ -57,35 +50,31 @@ fundamentals. In ZML, we describe a _Module_, which represents our AI model, as a Zig `struct`. That struct can contain Tensor fields that are used for computation, e.g. weights and biases. In the _forward_ function of a Module, we describe the -computation by calling tensor operations like _mul_, _add_, _dotGeneral_, +computation by calling tensor operations like _mul_, _add_, _dot_, _conv2D_, etc., or even nested Modules. ZML creates an MLIR representation of the computation when we compile the -Module. For compilation, only the _Shapes_ of all tensors must be known. No -actual tensor data is needed at this step. This is important for large models: -we can compile them while the actual weight data is being fetched from disk. +_Module_. For compilation, only the _Tensors_ are required. No actual tensor data +is needed at this step. This is important for large models: we can compile them +while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to -only load shapes and when to load actual tensor data. In our example, we will -fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays -instead. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything +that is required to make _Tensors_ and later materialize them to _Buffers_. In our +example though, we won't even use a _TensorStore_ as it's optional. We will directly +create tensors using their shape and data type. -After compilation is done (and the _BufferStore_ has finished loading weights), -we can send the weights from the _BufferStore_ to our computation device. That -produces an _executable_ module which we can call with different _inputs_. - -In our example, we then copy the result from the computation device to CPU -memory and print it. +After compilation is done, we get what we call an _Executable_. From this _Executable_ +we can create _Args_ and _Results_, two structs that will store respectively the inputs +and outputs of a computation done with an _Executable_. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- create a _BufferStore_ that provides _Shapes_ and data of weights and bias - (ca. 5 lines of code). -- compile the _Module_ **asynchronously** -- make the compiled _Module_ send the weights (and bias) to the computation - device utilizing the _BufferStore_, producing an _executable_ module -- prepare input tensor and call the _executable_ module. +- initialize the _Module_ +- compile the _Module_ to produce an _Executable_ +- make a "bufferized" version of the _Module_, containing the actual data on the + computation device +- prepare the _Args_ and _Results_ of a computation and call the _Executable_ - get the result back to CPU memory and print it If you like to read more about the underlying concepts of the above, please see @@ -99,13 +88,9 @@ Let's start by writing some Zig code, importing ZML and often-used modules: ```zig const std = @import("std"); const zml = @import("zml"); -const async = @import("async"); ``` -You will use above lines probably in all ZML projects. Also, note that **ZML is -async** and comes with its own async runtime, thanks to -[zigcoro](https://github.com/rsepassi/zigcoro). - +You will use above lines probably in all ZML projects. ### Defining our Model @@ -137,205 +122,139 @@ There are more things to observe: - more advanced use-cases are passing in / returning structs or tuples, like `struct { Tensor, Tensor }` as an example for a tuple of two tensors. You can see such use-cases, for example in the - [Llama Model](https://github.com/zml/zml/tree/master/examples/llama) + [LLM CLI](https://github.com/zml/zml/tree/master/examples/llm) - in the model, tensors may be optional. As is the case with `bias`. ### Adding a main() function -ZML code is async. Hence, We need to provide an async main function. It works -like this: - ```zig -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; - -pub fn asyncMain() !void { - // ... -``` - -The above `main()` function only creates an allocator and an async main thread -that executes our `asyncMain()` function by calling it with no (`.{}`) -arguments. - -So, let's start with the async main function: - -```zig -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + ... } ``` -This is boilerplate code that provides a general-purpose allocator and, for -convenience, an arena allocator that we will use later. The advantage of arena -allocators is that you don't need to deallocate individual allocations; you -simply call `.deinit()` to deinitialize the entire arena instead! +The above `main()` function takes advantage of Zig's "fat main" to avoid having to write +boilerplate code for allocations and io. -We also initialize the ZML context `context` and get our CPU `platform` -automatically. +We also get our ZML CPU platform `platform` automatically. +### Initializing the Module -### The BufferStore - -Next, we need to set up the concrete weight and bias tensors for our model. -Typically, we would load them from disk. But since our example works without -stored weights, we are going to create a BufferStore manually, containing -_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. - -A BufferStore basically contains a dictionary with string keys that match the -name of the struct fields of our `Layer` struct. So, let's create this -dictionary: +Next, we need to initialize the module so that we can compile it. ```zig -// Our weights and bias to use -var weights = [3]f16{ 2.0, 2.0, 2.0 }; -var bias = [3]f16{ 1.0, 2.0, 3.0 }; -const input_shape = zml.Shape.init(.{3}, .f16); - -// We manually produce a BufferStore. You would not normally do that. -// A BufferStore is usually created by loading model data from a file. -var buffers: zml.aio.BufferStore.Buffers = .{}; -try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); -try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - -// the actual BufferStore -const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, +const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; ``` -Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The -shape of the weight and bias tensors is `{3}`, and because of that, the **shape -of the input tensor** is also going to be `{3}`! - -Note that `zml.Shape` always takes the data type associated with the tensor. In -our example, that is `f16`, expressed as the enum value `.f16`. - +As you can see, it's pretty simple. You can create _Tensors_ from their shape and data type. +Alternatively, you are free to add an arbitrary `.init()` function to your _Module_ if the +initialization code is complex. ### Compiling our Module for the accelerator We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless. This compilation is usually done -asynchronously. That means, we can continue doing other things while the module -is compiling: +`forward()` function nonetheless: ```zig -// A clone of our model, consisting of shapes. We only need shapes for compiling. -// We use the BufferStore to infer the shapes. -const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); +// Our computation require an input tensor +const input: zml.Tensor = .init(.{3}, .f16); -// Start compiling. This uses the inferred shapes from the BufferStore. -// The shape of the input tensor, we have to pass in manually. -var compilation = try async.async( - zml.compileModel, - .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, -); - -// Produce a bufferized weights struct from the fake BufferStore. -// This is like the inferred shapes, but with actual values. -// We will need to send those to the computation device later. -var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); -defer zml.aio.unloadBuffers(&model_weights); // for good practice - -// Wait for compilation to finish -const compiled = try compilation.await(); +var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); +defer executable.deinit(); ``` -Compiling is happening in the background via the `async` function. We call -`async` with the `zml.compileModel` function and its arguments -separately. The arguments themselves are basically the shapes of the weights in -the BufferStore, the `.forward` function name in order to compile -`Layer.forward`, the shape of the input tensor(s), and the platform for which to -compile (we used auto platform). +You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, +and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. +For this simple example, we just replicate the tensors across all devices for simplicity. +### Creating the "bufferized" Model -### Creating the Executable Model - -Now that we have compiled the module utilizing the shapes, we turn it into an -executable. +Now that we have compiled the module, we can take care of loading the actual tensor data. +In our case, we'll create the _Buffers_ from the raw data directly (but keep in mind that ZML +provides ways to have extra-low loading times). ```zig -// pass the model weights to the compiled module to create an executable module -// all required memory has been allocated in `compile`. -var executable = compiled.prepare(model_weights); -defer executable.deinit(); + +const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); +const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); +var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), +}; +defer layer_buffers.weight.deinit(); +defer layer_buffers.bias.?.deinit(); ``` ### Calling / running the Model -The executable can now be invoked with an input of our choice. +Before being able to run our _Executable_ there is still two steps: +- create the input buffer +- create and fill the _Args_ and _Results_ structs -To create the `input`, we directly use `zml.Buffer` by calling -`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in -_accelerator_ (or _device_) memory, which is precisely where the input needs to -be for the executable to process it on the device. - -For clarity, let's recap the distinction: `HostBuffer`s are located in standard -_host_ memory, which is accessible by the CPU. When we initialized the weights, -we used `HostBuffers` to set up the `BufferStore`. This is because the -`BufferStore` typically loads weights from disk into `HostBuffer`s, and then -converts them into `Buffer`s when we call `loadBuffers()`. - -However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly -in device memory. +To create the `input` we already know how to do it, as we did it for the _Module_ +buffers. +To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig -// prepare an input buffer -// Here, we use zml.HostBuffer.fromSlice to show how you would create a -// HostBuffer with a specific shape from an array. -// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input -// shape. -var input = [3]f16{ 5.0, 5.0, 5.0 }; -var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), -); +// create the input buffer +const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); -// call our executable module -var result: zml.Buffer = executable.call(.{input_buffer}); +// create the Args and Results structs +var args = try executable.args(allocator); +defer args.deinit(allocator); + +var results = try executable.results(allocator); +defer results.deinit(allocator); + +// fill the Args +args.set(.{ layer_buffers, input_buffer }); + +// call our executable +executable.call(io, args, &results, .{}); + +// Retrieve the resulting buffer +var result = results.get(zml.Buffer); defer result.deinit(); // fetch the result buffer to CPU memory -const cpu_result = try result.toHostAlloc(arena); +const result_slice = try result.toSliceAlloc(allocator, io); +defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); ``` Note that the result of a computation is usually residing in the memory of the -computation device, so with `.toHostAlloc()` we bring it back to CPU memory in -the form of a `HostBuffer`. After that, we can print it. +computation device, so with `.toSliceAlloc()` we bring it back to CPU memory in +the form of a `Slice`. After that, we can print it. -In order to print it, we need to tell the host buffer how to interpret the -memory. We do that by calling `.items(f16)`, making it cast the memory to `f16` -items. +`call(io, args, &results, .{})` enqueues execution and returns immediately. Use +`call(io, args, &results, .{ .wait = true })` to wait for execution to finish before returning. +Here, `toSliceAlloc()` waits for the result before reading it. + +Raw calls leave input buffer handles owned by the caller, including inputs donated +with `reuseBuffer()`. Deinit their original handles after calling. Runners and +`eval` destroy donated input handles automatically. With runners, use output +destinations pointing to those buffers to replace them with the returned outputs. And that's it! Now, let's have a look at building and actually running this example! @@ -355,20 +274,18 @@ zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) ``` -To produce an executable, we import `zig_cc_binary` from the zig rules, and -pass it a name and the zig file we just wrote. The dependencies in `deps` are -what's needed for a basic ZML executable and correlate with our imports at the +To produce an executable, we import `zig_binary` from the zig rules, and +pass it a name and the zig file we just wrote. The dependency in `deps` is +what's needed for a basic ZML executable and correlate with our import at the top of the Zig file: ```zig const zml = @import("zml"); -const async = @import("async"); ``` @@ -381,41 +298,44 @@ With everything in place now, running the model is easy: bazel run --config=release //examples/simple_layer # compile and run debug version -bazel run //simple_layer +bazel run //examples/simple_layer ``` And voila! Here's the output: ``` -bazel run --config=release //simple_layer -INFO: Analyzed target //simple_layer:simple_layer (0 packages loaded, 0 targets configured). +bazel run --config=release //examples/simple_layer +INFO: Analyzed target //examples/simple_layer:simple_layer (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:simple_layer up-to-date: - bazel-bin/simple_layer/simple_layer +Target //examples/simple_layer:simple_layer up-to-date: + bazel-bin/examples/simple_layer/simple_layer INFO: Elapsed time: 0.120s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action -INFO: Running command line: bazel-bin/simple_layer/simple_layer -info(pjrt): Loaded library: libpjrt_cpu.dylib -info(zml_module): Compiling main.Layer.forward with { Shape({3}, dtype=.f16) } +INFO: Running command line: bazel-bin/examples/simple_layer/simple_layer +info(pjrt): Loaded library: libzml_cpu.dylib +debug(zml/module): +******** ZML generated MLIR ******** +module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { + sdy.mesh @replicated = <["bus"=4]> + func.func @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { + %0 = stablehlo.multiply %arg1, %arg2 : tensor<3xf16> + %1 = stablehlo.add %0, %arg0 : tensor<3xf16> + return %1 : tensor<3xf16> + } +} -The result of { 2, 2, 2 } * { 5, 5, 5 } + { 1, 2, 3 } = { 11, 12, 13 } + + +The result of {1,2,3} * {5,5,5} + {1,1,1} = {6,11,16} ``` ---- - -You can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - ## The complete example ```zig const std = @import("std"); const zml = @import("zml"); -const async = @import("async"); /// Model definition const Layer = struct { @@ -431,96 +351,68 @@ const Layer = struct { } }; -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try async.AsyncThread.main(gpa.allocator(), asyncMain); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(.{}); - - // Our weights and bias to use - var weights = [3]f16{ 2.0, 2.0, 2.0 }; - var bias = [3]f16{ 1.0, 2.0, 3.0 }; - const input_shape = zml.Shape.init(.{3}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; - // A clone of our model, consisting of shapes. We only need shapes for - // compiling. We use the BufferStore to infer the shapes. - const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); + // Our computation require an input tensor + const input: zml.Tensor = .init(.{3}, .f16); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); - - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice - - // Wait for compilation to finish - const compiled = try compilation.await(); - - // pass the model weights to the compiled module to create an executable - // module - var executable = compiled.prepare(model_weights); + var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); defer executable.deinit(); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a - // HostBuffer with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} - // input shape. - var input = [3]f16{ 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), - ); + const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); + const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); + var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + }; + defer layer_buffers.weight.deinit(); + defer layer_buffers.bias.?.deinit(); + + // create the input buffer + const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); - // call our executable module - var result: zml.Buffer = executable.call(.{input_buffer}); + // create the Args and Results structs + var args = try executable.args(allocator); + defer args.deinit(allocator); + + var results = try executable.results(allocator); + defer results.deinit(allocator); + + // fill the Args + args.set(.{ layer_buffers, input_buffer }); + + // call our executable + executable.call(io, args, &results, .{}); + + // Retrieve the resulting buffer + var result = results.get(zml.Buffer); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // fetch the result buffer to CPU memory + const result_slice = try result.toSliceAlloc(allocator, io); + defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); } ``` ## Where to go from here -- [Add some weights files to your model](../howtos/add_weights.md) - [Run the model on GPU](../tutorials/getting_started.md) - [Deploy the model on a server](../howtos/deploy_on_server.md) - [Dockerize this model](../howtos/dockerize_models.md) diff --git a/dsl/common/BUILD.bazel b/dsl/common/BUILD.bazel new file mode 100644 index 00000000..1476dcfd --- /dev/null +++ b/dsl/common/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "args.zig", + "control_flow.zig", + ], + import_name = "dsl/common/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/scf", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/common/args.zig b/dsl/common/args.zig new file mode 100644 index 00000000..84d57d49 --- /dev/null +++ b/dsl/common/args.zig @@ -0,0 +1,4 @@ +pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { + const field_names = @typeInfo(Spec).@"struct".field_names; + return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); +} diff --git a/dsl/common/builder.zig b/dsl/common/builder.zig new file mode 100644 index 00000000..1addd259 --- /dev/null +++ b/dsl/common/builder.zig @@ -0,0 +1,35 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +pub const control_flow = @import("control_flow.zig"); +pub const args = @import("args.zig"); + +pub const arity = control_flow.arity; +pub const BlockStack = control_flow.BlockStack; +pub const forBounds = control_flow.forBounds; +pub const ForOpts = scf.ForOpts; +pub const openFor = control_flow.openFor; +pub const openIf = control_flow.openIf; +pub const openIfElse = control_flow.openIfElse; +pub const openWhile = control_flow.openWhile; +pub const innerValues = control_flow.innerValues; +pub const resultTypes = control_flow.resultTypes; +pub const ForScope = control_flow.ForScope; +pub const IfOnlyScope = control_flow.IfOnlyScope; +pub const IfScope = control_flow.IfScope; +pub const WhileScope = control_flow.WhileScope; + +pub const NamedArgs = args.NamedArgs; + +/// Verifies `module`, logging it when it fails. +pub fn verify(module: *mlir.Module, comptime target: []const u8) error{InvalidMlir}!void { + if (module.operation().verify()) return; + std.log.err(target ++ ": the module does not verify:\n{f}", .{module.operation().fmt(.{ .debug_info = false })}); + return error.InvalidMlir; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/common/control_flow.zig b/dsl/common/control_flow.zig new file mode 100644 index 00000000..e557e24b --- /dev/null +++ b/dsl/common/control_flow.zig @@ -0,0 +1,364 @@ +//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`, +//! and `BlockStack`, the blocks a builder emits into. +//! +//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock` and +//! `currentBlock`. The openers also call its `lift` on literal inits and its +//! `scalarTy` on dtype result types, and `forBounds` its `constantLike` on +//! literal bounds. +//! `ValueT` must have an `inner: *const mlir.Value` field and a `kernel: ?*BuilderT` field. + +const std = @import("std"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +/// The blocks a builder emits into: the function's entry block at the bottom, +/// the innermost scope's on top. +pub const BlockStack = struct { + blocks: std.ArrayList(*mlir.Block) = .empty, + + pub fn push(self: *BlockStack, allocator: std.mem.Allocator, block: *mlir.Block) void { + self.blocks.append(allocator, block) catch @panic("BlockStack.push: out of memory"); + } + + /// Closes the innermost scope. + pub fn pop(self: *BlockStack) void { + if (self.blocks.items.len < 2) @panic("BlockStack.pop: no scope open"); // the entry block always stays + _ = self.blocks.pop(); + } + + /// Drops every block, keeping the memory for the next function. + pub fn clear(self: *BlockStack) void { + self.blocks.clearRetainingCapacity(); + } + + /// Returns the top block. + pub fn current(self: *const BlockStack) *mlir.Block { + return self.blocks.getLastOrNull() orelse @panic("BlockStack.current: no block, call declareArgs first"); + } + + /// Replaces the top block with `block`. + pub fn replaceTop(self: *BlockStack, block: *mlir.Block) void { + if (self.blocks.items.len == 0) @panic("BlockStack.replaceTop: no block, call declareArgs first"); + self.blocks.items[self.blocks.items.len - 1] = block; + } +}; + +/// The length of `T`, a tuple like `.{ v1, v2 }` or an array. +pub fn arity(comptime T: type, comptime what: []const u8) comptime_int { + return switch (@typeInfo(T)) { + .array => |a| a.len, + .@"struct" => |s| if (s.is_tuple) s.field_names.len else @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + else => @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + }; +} + +/// The MLIR values of `values`, `N` `ValueT`s. +pub fn innerValues(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { + if (arity(@TypeOf(values), what) != N) + @compileError(what ++ ": the arity must match the scope's"); + var buf: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + if (@TypeOf(values[i]) != ValueT) @compileError(what ++ ": every element must be a Value"); + buf[i] = values[i].inner; + } + return buf; +} + +/// `values` as `ValueT`s: values as they are, anything else through `kernel.lift`. +fn liftValues(kernel: anytype, comptime ValueT: type, values: anytype, comptime what: []const u8) [arity(@TypeOf(values), what)]ValueT { + var out: [arity(@TypeOf(values), what)]ValueT = undefined; + inline for (0..out.len) |i| out[i] = if (@TypeOf(values[i]) == ValueT) values[i] else kernel.lift(values[i]); + return out; +} + +/// The types `types` names: `*const mlir.Type`s, `ValueT`s for their types, +/// or dtypes (`.f32`) for `kernel.scalarTy`. +pub fn resultTypes(kernel: anytype, comptime ValueT: type, types: anytype, comptime what: []const u8) [arity(@TypeOf(types), what)]*const mlir.Type { + var out: [arity(@TypeOf(types), what)]*const mlir.Type = undefined; + inline for (0..out.len) |i| { + const T = @TypeOf(types[i]); + out[i] = if (T == *const mlir.Type) types[i] else if (T == ValueT) types[i].inner.type_() else kernel.scalarTy(types[i]); + } + return out; +} + +fn scfYield(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, values: anytype, comptime what: []const u8) void { + const buf = innerValues(ValueT, N, values, what); + _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); +} + +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + opts: scf.ForOpts = .{}, + + const Self = @This(); + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.body, values, "ForScope.yield"); + k.popBlock(); + const for_op = scf.for_( + k.ctx, + self.lb_inner, + self.ub_inner, + self.step_inner, + &self.inits_inner, + self.body, + self.opts, + k.loc(), + ); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn close(self: *Self) void { + const k = self.kernel; + scfYield(ValueT, 0, k, self.then_block, .{}, "IfOnlyScope.close"); + k.popBlock(); + const if_op = scf.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.then_block, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + fn endElse(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.else_block, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = scf.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize, comptime M: usize) type { + return struct { + kernel: *BuilderT, + before_block: *mlir.Block, + after_block: *mlir.Block, + inits_inner: [N]*const mlir.Value, + after_types: [M]*const mlir.Type, + before_carried: [N]ValueT, + after_carried: [M]ValueT = undefined, + results: [M]ValueT = undefined, + + const Self = @This(); + + pub fn yieldBefore(self: *Self, cond: ValueT, forwarded: anytype) void { + const k = self.kernel; + const buf = innerValues(ValueT, M, forwarded, "WhileScope.yieldBefore"); + _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); + k.popBlock(); + k.pushBlock(self.after_block); + for (0..M) |i| self.after_carried[i] = .{ .inner = self.after_block.argument(i), .kernel = k }; + } + + pub fn yieldAfter(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.after_block, values, "WhileScope.yieldAfter"); + k.popBlock(); + const w = scf.while_( + k.ctx, + &self.inits_inner, + &self.after_types, + self.before_block, + self.after_block, + k.loc(), + ); + _ = w.appendTo(k.currentBlock()); + for (0..M) |i| self.results[i] = .{ .inner = w.result(i), .kernel = k }; + } + }; +} + +/// `openFor`'s bounds as values. A bound that isn't a value takes the type of +/// the first one that is (`kernel.constantLike`); with none, `lower` becomes +/// `defaultLower(kernel, lower)` and the others take its type. +pub fn forBounds( + kernel: anytype, + comptime ValueT: type, + lower: anytype, + upper: anytype, + step: anytype, + comptime defaultLower: anytype, +) [3]ValueT { + const bounds: [3]ValueT = if (@TypeOf(lower) != ValueT and @TypeOf(upper) != ValueT and @TypeOf(step) != ValueT) blk: { + const lb = defaultLower(kernel, lower); + break :blk .{ lb, kernel.constantLike(upper, lb), kernel.constantLike(step, lb) }; + } else blk: { + const first: ValueT = if (@TypeOf(lower) == ValueT) lower else if (@TypeOf(upper) == ValueT) upper else step; + break :blk .{ kernel.constantLike(lower, first), kernel.constantLike(upper, first), kernel.constantLike(step, first) }; + }; + const t = bounds[0].inner.type_(); + if (!bounds[1].inner.type_().eql(t) or !bounds[2].inner.type_().eql(t)) + std.debug.panic("openFor: bounds of types {f}, {f} and {f}", .{ t, bounds[1].inner.type_(), bounds[2].inner.type_() }); + return bounds; +} + +/// Opens `for iv in [bounds[0], bounds[1]) step bounds[2]` carrying `inits`, +/// values or literals `kernel.lift` takes. +pub fn openFor( + kernel: anytype, + comptime ValueT: type, + bounds: [3]ValueT, + inits: anytype, + opts: scf.ForOpts, +) ForScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime arity(@TypeOf(inits), "openFor: inits"); + const values = liftValues(kernel, ValueT, inits, "openFor: inits"); + var types: [N + 1]*const mlir.Type = undefined; + types[0] = bounds[0].inner.type_(); + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + types[i + 1] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const locs: [N + 1]*const mlir.Location = @splat(kernel.loc()); + const body = mlir.Block.init(&types, &locs); + kernel.pushBlock(body); + var carried: [N]ValueT = undefined; + for (&carried, 0..) |*c, i| c.* = .{ .inner = body.argument(i + 1), .kernel = kernel }; + return .{ + .kernel = kernel, + .body = body, + .lb_inner = bounds[0].inner, + .ub_inner = bounds[1].inner, + .step_inner = bounds[2].inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = kernel }, + .carried = carried, + .opts = opts, + }; +} + +pub fn openIf(kernel: anytype, comptime ValueT: type, cond: ValueT) IfOnlyScope(@TypeOf(kernel.*), ValueT) { + const then_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ .kernel = kernel, .cond_inner = cond.inner, .then_block = then_block }; +} + +/// Opens an `if` with an `else`, of results of `result_types` (see `resultTypes`). +pub fn openIfElse( + kernel: anytype, + comptime ValueT: type, + cond: ValueT, + result_types: anytype, +) IfScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(result_types), "openIfElse: result_types")) { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ + .kernel = kernel, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = resultTypes(kernel, ValueT, result_types, "openIfElse: result_types"), + }; +} + +/// Opens an `scf.while` carrying `inits` into its condition region, which +/// forwards values of `after_types` (see `resultTypes`) to its body. +pub fn openWhile( + kernel: anytype, + comptime ValueT: type, + inits: anytype, + after_types: anytype, +) WhileScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openWhile: inits"), arity(@TypeOf(after_types), "openWhile: after_types")) { + const N = comptime arity(@TypeOf(inits), "openWhile: inits"); + const M = comptime arity(@TypeOf(after_types), "openWhile: after_types"); + const values = liftValues(kernel, ValueT, inits, "openWhile: inits"); + var before_types: [N]*const mlir.Type = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + before_types[i] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const before_locs: [N]*const mlir.Location = @splat(kernel.loc()); + const before_block = mlir.Block.init(&before_types, &before_locs); + const after_types_ = resultTypes(kernel, ValueT, after_types, "openWhile: after_types"); + const after_locs: [M]*const mlir.Location = @splat(kernel.loc()); + const after_block = mlir.Block.init(&after_types_, &after_locs); + kernel.pushBlock(before_block); + var before_carried: [N]ValueT = undefined; + for (&before_carried, 0..) |*c, i| c.* = .{ .inner = before_block.argument(i), .kernel = kernel }; + return .{ + .kernel = kernel, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_types_, + .before_carried = before_carried, + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/cuda_tile/BUILD.bazel b/dsl/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..527c53e4 --- /dev/null +++ b/dsl/cuda_tile/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "control_flow.zig", + "dtype.zig", + ], + import_name = "dsl/cuda_tile/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/cuda_tile", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/cuda_tile/README.md b/dsl/cuda_tile/README.md new file mode 100644 index 00000000..d8c5fdb8 --- /dev/null +++ b/dsl/cuda_tile/README.md @@ -0,0 +1,137 @@ +# CUDA Tile IR kernels in Zig + +`dsl/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect +(CUDA Tile IR). It is the third instance of the `dsl/triton` / +`dsl/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent arithmetic, and a +`zml.kernel.cuda_tile.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles with `tileiras`. + +The dialect binding lives in `mlir/dialects/cuda_tile`: one function per op +(all 100 of them, in `Ops.td` order), every type through the dialect's C API, +every enum attribute through its C getter. + +## A kernel + +```zig +const zml = @import("zml"); +const cut = zml.kernel.cuda_tile; + +const Cfg = struct { n: i64 }; + +pub const AddOne = cut.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *cut.Builder, cfg: Cfg) cut.FinishError!void { + // Every parameter is a rank-0 tile>; views are built inside. + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + // pointer -> tensor_view -> partition_view; the view pads the ragged edge. + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); +} + +// In a model: +const out = AddOne.call(.{ .x = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .n = 1024 }, + .grid = .{ 8, 1, 1 }, +}).out; +``` + +Put each scope's body in a Zig `{ ... }` block right after `openFor`, `openIf` +or `openLoop`, ending with its `yield` (or `close()`). `openIfElse` gets one +block per branch, ending with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` +without results). + +## What is different from the Triton DSL + +* **Rank 0 is the scalar.** Every value is a `tile<...>`; `entry` parameters + must be rank 0, so buffers arrive as `tile>`. +* **Memory is a view stack.** `tensorView` → `partitionView` (or + `stridedView`, `gatherScatterView`) → tile-space `load`/`store`. A partial + edge tile reads the view's `padding` and its store is masked; there is no + mask tile. The pointer-tile arm (`loadPtr`/`storePtr` with `offset`) exists + for gathers but cannot use TMA. +* **Every load and store returns a token.** The dialect has no barrier or + fence; `loadOpts`/`storeOpts` take and return tokens, `load`/`store` drop + them. +* **Float ops carry a rounding mode.** `addf` defaults to `nearest_even`; + `addfOpts` sets it. Integer ops carry none. +* **Loop bodies end with `continue`, `if` bodies with `yield`, and `loop` + exits with `break`** — `ForScope`, `IfScope`, `LoopScope` do this for you. +* **No `num_warps`/`num_stages`.** Warp and CTA tuning is + `optimization_hints` on the entry, per architecture (`Opts.hints`), and + `allow_tma`/`latency` on view loads and stores (`LoadOpts.hints`). +* **Tile dims are powers of two**, at most 2^24 elements; `tileTy` and + `partitionView` check this in Zig. +* **Some conversions are one-way.** `ftof` to `f8e8m0fnu` (the MX scale + type) rounds toward zero by default (`CastOpts.rounding = .positive_inf` + is the other legal choice); an integer reaches it through `f32`, which + `cast`/`to` do for you. `ftoi` always truncates toward zero. `tanh` takes + only `approx`/`full`. `Value.cdiv` is integer-only. +* **`i4` is a tile element type, not a pointee.** Declare a 4-bit buffer as + `.i8` and `unpack` it in the kernel. +* **`atomicRedView` is relaxed-only** on an unpadded view and takes no mask; + use the pointer atomics (`atomicRmw`, `atomicCas`) for masked updates. +* **Reductions over several tiles at once** (argmax, an online softmax's + (max, sum)) go through `reduceMulti`/`scanMulti`. +* **A scalar argument is a pointer**: `.{ .n = .{ .ptr = .i32 } }` then + `b.loadPtr(a.n)` gives the rank-0 tile; XLA passes device addresses only. + +## The custom call + +`ops.cudaTile` emits `__gpu$xla.gpu.cuda_tile` with a printed dictionary: +`name`, `kernel_type = "cuda_tile"`, `ir` (the textual module), `grid_x/y/z`, +optional `ir_version`. XLA parses the module, renames the entry, checks the +argument count against the operands + result leaves, scrubs locations, +serializes bytecode, runs `tileiras`, and launches over `grid` tile blocks with +a `(1,1,1)` block and no shared memory — cuda-tile's launch ABI. Every operand +and result is one raw device pointer in the default layout, operands first; +XLA rejects an entry whose parameters are not all `tile>`. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cuda_tile.mlir` (post-rename) and `.tilebc` (the exact bytes +handed to `tileiras`). + +## Finding what exists + +There is one function per op and no hidden API, so grep is the index: + +```sh +# every op of the dialect, in Ops.td order (115 constructors) +grep -n '^pub fn ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the types, enums and attribute helpers they take +grep -n '^pub const ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the DSL: Value's fluent methods (add, mul, lt, to, sum, reshape, ...) +awk '/^pub const Value = struct/,/^};/' dsl/cuda_tile/builder.zig | grep 'pub fn ' + +# the DSL: everything on Builder (views, load/store, math, casts, reductions, +# atomics, control flow) -- the `*Opts` variant of a call takes the options +awk '/^pub const Builder = struct/,0' dsl/cuda_tile/builder.zig | grep ' pub fn ' + +# what each option struct holds, and its defaults +grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig \ + dsl/cuda_tile/control_flow.zig + +# for / if / loop scopes and their terminators +grep -n 'pub fn ' dsl/cuda_tile/control_flow.zig + +# element types +grep -n 'pub const DType = enum' -A 30 dsl/cuda_tile/dtype.zig +``` + +Every op also has a test: `grep -n '^test "' dsl/cuda_tile/builder.zig` +lists them, and each one prints IR that round-trips through the verifier, so +they double as usage examples. diff --git a/dsl/cuda_tile/builder.zig b/dsl/cuda_tile/builder.zig new file mode 100644 index 00000000..a8414acc --- /dev/null +++ b/dsl/cuda_tile/builder.zig @@ -0,0 +1,2218 @@ +const std = @import("std"); + +const common = @import("dsl/common/builder"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const cf = @import("control_flow.zig"); +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +pub const RoundingMode = cuda_tile.RoundingMode; +pub const Signedness = cuda_tile.Signedness; +pub const IntegerOverflow = cuda_tile.IntegerOverflow; +pub const ComparisonPredicate = cuda_tile.ComparisonPredicate; +pub const ComparisonOrdering = cuda_tile.ComparisonOrdering; +pub const AtomicRMWMode = cuda_tile.AtomicRMWMode; +pub const MemoryScope = cuda_tile.MemoryScope; +pub const MemoryOrdering = cuda_tile.MemoryOrdering; +pub const PaddingValue = cuda_tile.PaddingValue; +pub const Arch = cuda_tile.Arch; +pub const EntryHint = cuda_tile.EntryHint; +pub const LoadStoreHint = cuda_tile.LoadStoreHint; +pub const PtrLoadStoreHint = cuda_tile.PtrLoadStoreHint; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{cuda_tile}) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn tile(self: Value) *const cuda_tile.TileType { + return self.type_().isA(cuda_tile.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); + } + + pub fn isTile(self: Value) bool { + return self.type_().isA(cuda_tile.TileType) != null; + } + + pub fn elemType(self: Value) *const mlir.Type { + return self.tile().elementType(); + } + + pub fn rank(self: Value) usize { + return self.tile().rank(); + } + + pub fn isScalar(self: Value) bool { + return self.rank() == 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.tile().dimension(i); + } + + pub fn shape(self: Value) Value.Shape { + var out: Value.Shape = .empty; + const t = self.tile(); + for (0..t.rank()) |i| out.appendAssumeCapacity(t.dimension(i)); + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn isPtrElem(self: Value) bool { + return self.elemType().isA(cuda_tile.PointerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divi(l, r, .signed); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remi(l, r, .signed); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloatElem()) @panic("Value.cdiv is integer-only; use div (and ceil) on floats"); + return k.diviOpts(l, r, .{ .signedness = .signed, .rounding = .positive_inf }); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.negf(self) else k.negi(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.absf(self) else k.absi(self); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.xori(l, r); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shli(l, r); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shri(l, r, .signed); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + fn compare(self: Value, rhs: anytype, pred: ComparisonPredicate) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(pred, l, r) else k.cmpi(pred, l, r, .signed); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than); + } + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than_or_equal); + } + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than); + } + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than_or_equal); + } + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(rhs, .equal); + } + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(rhs, .not_equal); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + + pub fn reshape(self: Value, shape_: []const i64) Value { + return self.kern().reshape(self, shape_); + } + + pub fn broadcastTo(self: Value, shape_: []const i64) Value { + return self.kern().broadcastTo(self, shape_); + } + + pub fn expandDims(self: Value, axis: usize) Value { + return self.kern().expandDims(self, axis); + } + + pub fn permute(self: Value, order: []const i32) Value { + return self.kern().permute(self, order); + } + + pub fn sum(self: Value, axis: usize) Value { + return self.kern().sum(self, axis); + } + + pub fn max(self: Value, axis: usize) Value { + return self.kern().max(self, axis); + } + + pub fn min(self: Value, axis: usize) Value { + return self.kern().min(self, axis); + } + + pub fn cumsum(self: Value, axis: usize) Value { + return self.kern().cumsum(self, axis); + } + + pub fn offset(self: Value, off: anytype) Value { + const k = self.kern(); + const o: Value = if (@TypeOf(off) == Value) off else k.lift(off); + const p, const o2 = k.coerceShapes(self, o); + return k.offset(p, o2); + } +}; + +pub const Loaded = struct { + tile: Value, + token: Value, +}; + +pub const BlockId = struct { x: Value, y: Value, z: Value }; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument, so every + /// parameter is a rank-0 `tile>` and views are built inside the + /// kernel. A scalar reaches the kernel through a pointer (`loadPtr` on a + /// rank-0 pointer tile) or as a comptime constant in `cfg`. + pub const Kind = union(enum) { + ptr: DType, + ptr_opts: PtrOpts, + }; + + /// `div_by` becomes a `cuda_tile.assume #cuda_tile.div_by` at the top + /// of the entry, and `arg(i)` returns the refined value. XLA guarantees + /// only 16 bytes on the pointer it passes. + pub const PtrOpts = struct { + dtype: DType, + div_by: ?u32 = 16, + }; +}; + +pub const Opts = struct { + hints: []const EntryHint = &.{}, +}; + +pub const FloatOpts = struct { + rounding: RoundingMode = .nearest_even, + flush_to_zero: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: bool = false, + flush_to_zero: bool = false, +}; + +pub const IntOpts = struct { + overflow: IntegerOverflow = .none, +}; + +pub const DivIOpts = struct { + signedness: Signedness = .signed, + rounding: ?RoundingMode = null, +}; + +/// `rounding` null picks the target's legal default: `nearest_even`, except +/// `f8e8m0fnu`, which the dialect only lets you reach with `zero` or +/// `positive_inf`. +pub const CastOpts = struct { + rounding: ?RoundingMode = null, + signedness: Signedness = .signed, +}; + +/// `ftoi` admits exactly one rounding mode (`nearest_int_to_zero`), so only +/// the signedness is a choice. +pub const FToIOpts = struct { + signedness: Signedness = .signed, +}; + +pub const MmaOpts = struct { + /// 13.3 only. + fast_acc: bool = false, +}; + +pub const PartitionOpts = struct { + padding: ?PaddingValue = .zero, + dim_map: []const i32 = &.{}, +}; + +pub const LoadOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const LoadStoreHint = &.{}, +}; + +pub const StoreOpts = LoadOpts; + +/// The pointer arm takes `latency` hints only; `allow_tma` is a view-op hint. +pub const LoadPtrOpts = struct { + mask: ?Value = null, + padding: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const StorePtrOpts = struct { + mask: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?Value = null, + token: ?Value = null, +}; + +/// `atomic_red_view_tko` is relaxed-only, scoped to the block or device, and +/// has no mask operand — the view's padding decides what is touched. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?Value = null, +}; + +pub const AllocaOpts = struct { + alignment: i64 = 16, + global: bool = false, +}; + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?cuda_tile.SymbolVisibility = null, +}; + +pub const ScanOpts = struct { + reverse: bool = false, +}; + +pub fn ForScope(comptime N: usize) type { + return cf.ForScope(Builder, Value, N); +} + +pub const ForOpts = cf.ForOpts; + +pub const IfOnlyScope = cf.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return cf.IfScope(Builder, Value, N); +} + +pub fn LoopScope(comptime N: usize) type { + return cf.LoopScope(Builder, Value, N); +} + +/// A custom reduction: `combine(builder, element, accumulator, ctx)` runs on +/// rank-0 tiles of `src`'s element type; `identity` is the element attribute +/// the dialect requires (see `Builder.identityAttr`). +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + reverse: bool = false, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +/// A reduction over several same-shaped tiles at once (argmax, an online +/// softmax's (max, sum)): `combine(builder, elements, accumulators, ctx)` +/// gets one rank-0 tile per source in each slice and returns the new +/// accumulators, one per source; `identities` is one element attribute per +/// source. +pub fn ReduceMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub fn ScanMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + reverse: bool = false, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + block_stack: common.BlockStack = .{}, + ct_module: ?*mlir.Operation = null, + ct_body: ?*mlir.Block = null, + refined_args: []?Value = &.{}, + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + ) !Builder { + return initOpts(allocator, ctx, name, args, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); + } + + pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (0.., field_names) |i, field_name| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .div_by = if (@hasField(@TypeOf(inner), "div_by")) inner.div_by else 16, + } }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (0.., field_names) |i, field_name| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts(self: *Builder, args: []const ArgSpec, opts: Opts) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt), + .ptr_opts => |p| self.ptrTy(p.dtype), + }; + a_loc.* = unknown_loc; + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const ct_body = mlir.Block.init(&.{}, &.{}); + const ct_module = cuda_tile.module(ctx, self.name, ct_body, unknown_loc); + _ = ct_module.appendTo(self.module.body()); + + const func_op = cuda_tile.entry(ctx, .{ + .name = self.name, + .block = entry, + .optimization_hints = if (opts.hints.len > 0) cuda_tile.optimizationHintsAttr(ctx, opts.hints) else null, + .location = unknown_loc, + }); + _ = func_op.appendTo(ct_body); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + self.ct_module = ct_module; + self.ct_body = ct_body; + + self.refined_args = try scratch.alloc(?Value, args.len); + for (self.refined_args, args, 0..) |*refined, a, i| { + refined.* = switch (a.kind) { + .ptr => self.assumeDivBy(self.rawArg(i), 16), + .ptr_opts => |p| if (p.div_by) |n| self.assumeDivBy(self.rawArg(i), n) else null, + }; + } + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// Parameter `i`, refined by its `assume` when one was declared. The + /// owner is re-stamped: `initOpts` fills `refined_args` through a Builder + /// that is then moved out of its frame. + pub fn arg(self: *Builder, i: usize) Value { + if (i < self.refined_args.len) { + if (self.refined_args[i]) |v| return .{ .inner = v.inner, .kernel = self }; + } + return self.rawArg(i); + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + fn innerOpt(v: ?Value) ?*const mlir.Value { + return if (v) |x| x.inner else null; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(T, f) != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); + out[i] = @field(values, f); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// `tile`; `&.{}` for a scalar. + pub fn tileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (!dtypes.tileLegal(shape)) { + std.debug.panic("Builder.tileTy: {any} is not a legal tile shape (power-of-two dims, at most 2^24 elements)", .{shape}); + } + return cuda_tile.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.tileTy(&.{}, dtype); + } + + /// `tile>` — the type of a buffer parameter. + pub fn ptrTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.ptrTileTy(&.{}, dtype); + } + + /// `tile>`. `i4` is a tile element type but not a pointee + /// (Types.td: CudaTile_NumberType excludes Int4), so a 4-bit buffer is + /// declared as `.i8` and `unpack`ed in the kernel. + pub fn ptrTileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (dtype == .i4) @panic("Builder: i4 is a tile element type only, never a pointee; declare the buffer as .i8 and unpack in-kernel"); + return cuda_tile.tileType(self.ctx, shape, cuda_tile.pointerType(self.ctx, dtype.toMlir(self.ctx))); + } + + pub fn tokenTy(self: *const Builder) *const mlir.Type { + return cuda_tile.tokenType(self.ctx); + } + + fn tileLike(self: *const Builder, v: Value, elem: *const mlir.Type) *const mlir.Type { + return cuda_tile.tileType(self.ctx, v.shape().constSlice(), elem); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + fn pointee(v: Value) *const mlir.Type { + const p = v.elemType().isA(cuda_tile.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); + return p.pointee(); + } + + fn elemAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + const elem = dtype.toMlir(self.ctx); + const T = @TypeOf(value); + if (isFloatDtype(dtype)) { + const v: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return cuda_tile.floatElemAttr(self.ctx, elem, v); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return cuda_tile.intElemAttr(elem, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); + } + + /// `value` as a rank-0 constant of `dtype`; a Value passes through. An + /// integer is signed or an unsigned bit pattern of `dtype`'s width; + /// anything else panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + return self.splat(self.scalarTy(dtype), dtype, value); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.tileTy(shape, dtype), dtype, value); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + /// A dense constant from a slice of Zig scalars in row-major order. MLIR + /// stores every element in whole bytes except `i1`, which it bit-packs; + /// the slice must hold exactly that many bytes (use `full` for a splat). + pub fn dense(self: *Builder, shape: []const i64, dtype: DType, values: anytype) Value { + checkDenseBytes("Builder.dense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.denseAttr(ty, values), ty, self.loc())); + } + + fn checkDenseBytes(comptime who: []const u8, shape: []const i64, dtype: DType, values: anytype) void { + var elems: usize = 1; + for (shape) |d| elems *= @intCast(d); + const bits = dtypeBitwidth(dtype); + const expected: usize = if (bits == 1) (elems + 7) / 8 else ((bits + 7) / 8) * elems; + const got = std.mem.sliceAsBytes(values).len; + if (got != expected) { + std.debug.panic("{s}: {d} bytes given for {any} x {s}, {d} expected", .{ who, got, shape, @tagName(dtype), expected }); + } + } + + pub fn iota(self: *Builder, n: i64, dtype: DType) Value { + return self.emit(cuda_tile.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); + } + + /// The element attribute `reduce`/`scan` need as an identity. + pub fn identityAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + return self.elemAttr(dtype, value); + } + + /// Lift a Zig scalar to a rank-0 constant. Dtype follows the source type; + /// unsigned ints are bit-cast. Use `constant` for an explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.constant(.i32, value); + break :blk self.constant(.i64, value); + }, + .int => |info| if (info.bits > 32) self.constant(.i64, value) else self.constant(.i32, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constant(self.mlirElemToDType(ref_elem), value), + else => self.lift(value), + }; + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.type_()); + } + + fn i32Bound(self: *Builder, value: anytype) Value { + return self.constant(.i32, value); + } + + fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { + const elem = if (ref.isA(cuda_tile.TileType)) |t| t.elementType() else ref; + return self.constant(self.mlirElemToDType(elem), value); + } + + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + return self.expandTo(v, ref.shape().constSlice()); + } + + fn expandTo(self: *Builder, v: Value, target: []const i64) Value { + const cur = v.shape(); + if (std.mem.eql(i64, cur.constSlice(), target)) return v; + var aligned = v; + if (cur.len != target.len) { + std.debug.assert(cur.len <= target.len); + var padded: Value.Shape = .empty; + for (0..target.len - cur.len) |_| padded.appendAssumeCapacity(1); + padded.appendSliceAssumeCapacity(cur.constSlice()); + aligned = self.reshape(v, padded.constSlice()); + } + if (std.mem.eql(i64, aligned.shape().constSlice(), target)) return aligned; + return self.broadcastTo(aligned, target); + } + + fn coerceShapes(self: *Builder, a: Value, b: Value) struct { Value, Value } { + const a_sh = a.shape(); + const b_sh = b.shape(); + if (std.mem.eql(i64, a_sh.constSlice(), b_sh.constSlice())) return .{ a, b }; + const n = @max(a_sh.len, b_sh.len); + var target: Value.Shape = .empty; + for (0..n) |i| { + const ai: i64 = if (i + a_sh.len >= n) a_sh.get(i + a_sh.len - n) else 1; + const bi: i64 = if (i + b_sh.len >= n) b_sh.get(i + b_sh.len - n) else 1; + if (ai == bi or bi == 1) { + target.appendAssumeCapacity(ai); + } else if (ai == 1) { + target.appendAssumeCapacity(bi); + } else { + std.debug.panic("Builder.coerce: incompatible shapes {any} vs {any} at axis {d}", .{ a_sh.constSlice(), b_sh.constSlice(), i }); + } + } + return .{ self.expandTo(a, target.constSlice()), self.expandTo(b, target.constSlice()) }; + } + + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + return self.coerceShapes(av, bv); + } + + pub fn tileBlockId(self: *Builder) BlockId { + const r = self.emitMulti(cuda_tile.get_tile_block_id(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn numTileBlocks(self: *Builder) BlockId { + const r = self.emitMulti(cuda_tile.get_num_tile_blocks(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn assumeDivBy(self: *Builder, v: Value, n: u64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{}), self.loc())); + } + + pub fn assumeDivByEvery(self: *Builder, v: Value, n: u64, every: i64, along: i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{ .every = every, .along = along }), self.loc())); + } + + pub fn assumeSameElements(self: *Builder, v: Value, values: []const i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.sameElementsAttr(self.ctx, values), self.loc())); + } + + pub fn assumeBounded(self: *Builder, v: Value, lb: ?i64, ub: ?i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.boundedAttr(self.ctx, .{ .lb = lb, .ub = ub }), self.loc())); + } + + pub fn tensorView(self: *Builder, base: Value, shape: []const i64, strides: []const i64) Value { + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), shape, strides); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); + } + + pub const Dim = union(enum) { + static: i64, + dynamic: Value, + }; + + pub fn tensorViewDyn(self: *Builder, base: Value, shape: []const Dim, strides: []const Dim) Value { + const scratch = self.arena.allocator(); + var static_shape: Value.Shape = .empty; + var static_strides: Value.Shape = .empty; + var dyn_shape = std.ArrayList(*const mlir.Value).empty; + var dyn_strides = std.ArrayList(*const mlir.Value).empty; + for (shape) |d| switch (d) { + .static => |s| static_shape.appendAssumeCapacity(s), + .dynamic => |v| { + static_shape.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); + dyn_shape.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + for (strides) |d| switch (d) { + .static => |s| static_strides.appendAssumeCapacity(s), + .dynamic => |v| { + static_strides.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); + dyn_strides.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); + } + + fn i32Slice(self: *Builder, values: []const i64) []const i32 { + const out = self.arena.allocator().alloc(i32, values.len) catch @panic("OOM"); + for (values, 0..) |v, i| out[i] = @intCast(v); + return out; + } + + /// Tiles `tv` into `tile`-shaped pieces addressed by tile-space indices. + /// `padding` fills the ragged edge on load; stores past the edge are + /// masked. That is why this DSL has no mask tile. + pub fn partitionView(self: *Builder, tv: Value, tile: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.partitionView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(self.ctx, .{ .tile_shape = self.i32Slice(tile), .tensor_view = tv.type_(), .dim_map = opts.dim_map, .padding = opts.padding })); + return self.emit(cuda_tile.make_partition_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: like `partitionView`, stepping by `traversal_strides` elements + /// per index instead of a whole tile. + pub fn stridedView(self: *Builder, tv: Value, tile: []const i64, traversal_strides: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.stridedView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(cuda_tile.StridedViewType.get(self.ctx, .{ + .tile_shape = self.i32Slice(tile), + .traversal_strides = self.i32Slice(traversal_strides), + .tensor_view = tv.type_(), + .dim_map = opts.dim_map, + .padding = opts.padding, + })); + return self.emit(cuda_tile.make_strided_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: a view whose `sparse_dim` is indexed by a rank-1 tile of + /// `tile[sparse_dim]` indices (every other index stays rank 0). + pub fn gatherScatterView(self: *Builder, tv: Value, tile: []const i64, sparse_dim: u32, padding: ?PaddingValue) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.gatherScatterView: {any} is not a legal tile shape", .{tile}); + const ty = cuda_tile.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, .{ .padding = padding }); + return self.emit(cuda_tile.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); + } + + fn i32Scalars(self: *Builder, n: usize) []const *const mlir.Type { + const out = self.arena.allocator().alloc(*const mlir.Type, n) catch @panic("OOM"); + for (out) |*t| t.* = self.scalarTy(.i32); + return out; + } + + pub fn indexSpaceShape(self: *Builder, view: Value) []Value { + const n = cuda_tile.indexRankOfView(view.type_()); + return self.emitMulti(cuda_tile.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); + } + + pub fn tensorShape(self: *Builder, tv: Value) []Value { + const t = tv.type_().isA(cuda_tile.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); + const n = t.rank(); + return self.emitMulti(cuda_tile.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); + } + + fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) cuda_tile.MemOpts { + return .{ + .ordering = ordering, + .scope = scope, + .token = innerOpt(token), + .optimization_hints = if (hints.len > 0) cuda_tile.optimizationHintsAttr(self.ctx, hints) else null, + }; + } + + pub fn load(self: *Builder, view: Value, index: []const Value) Value { + return self.loadOpts(view, index, .{}).tile; + } + + pub fn loadOpts(self: *Builder, view: Value, index: []const Value, opts: LoadOpts) Loaded { + const expected = cuda_tile.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.load: view takes {d} indices, got {d}", .{ expected, index.len }); + const r = self.emitMulti(cuda_tile.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn store(self: *Builder, tile: Value, view: Value, index: []const Value) Value { + return self.storeOpts(tile, view, index, .{}); + } + + pub fn storeOpts(self: *Builder, tile: Value, view: Value, index: []const Value, opts: StoreOpts) Value { + const expected = cuda_tile.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.store: view takes {d} indices, got {d}", .{ expected, index.len }); + return self.emit(cuda_tile.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); + } + + /// Gather through a tile of pointers, dropping the token. This arm cannot + /// use TMA; prefer views for dense access. + pub fn loadPtr(self: *Builder, ptrs: Value) Value { + return self.loadPtrOpts(ptrs, .{}).tile; + } + + pub fn loadPtrOpts(self: *Builder, ptrs: Value, opts: LoadPtrOpts) Loaded { + const result_ty = self.tileLike(ptrs, pointee(ptrs)); + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + const r = self.emitMulti(cuda_tile.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ + .mask = innerOpt(mask), + .padding = innerOpt(opts.padding), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn storePtr(self: *Builder, ptrs: Value, value: Value) Value { + return self.storePtrOpts(ptrs, value, .{}); + } + + pub fn storePtrOpts(self: *Builder, ptrs: Value, value: Value, opts: StorePtrOpts) Value { + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + return self.emit(cuda_tile.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ + .mask = innerOpt(mask), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc())); + } + + pub fn makeToken(self: *Builder) Value { + return self.emit(cuda_tile.make_token(self.ctx, self.loc())); + } + + pub fn joinTokens(self: *Builder, tokens: []const Value) Value { + return self.emit(cuda_tile.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); + } + + pub fn print(self: *Builder, str: []const u8, args: []const Value) Value { + return self.printOpts(str, args, null); + } + + pub fn printOpts(self: *Builder, str: []const u8, args: []const Value, token: ?Value) Value { + return self.emit(cuda_tile.print_tko(self.ctx, str, self.innerSlice(args), .{ .token = innerOpt(token) }, self.loc())); + } + + fn atomicOpts(opts: AtomicOpts) cuda_tile.AtomicOpts { + return .{ .ordering = opts.ordering, .scope = opts.scope, .mask = innerOpt(opts.mask), .token = innerOpt(opts.token) }; + } + + pub fn atomicCas(self: *Builder, ptrs: Value, cmp: Value, val: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(cuda_tile.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn atomicRmw(self: *Builder, mode: AtomicRMWMode, ptrs: Value, operand: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(cuda_tile.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + /// 13.3: an atomic reduction into an unpadded tiled view; returns the + /// token. `xchg` is not a reduction and the dialect rejects it. + pub fn atomicRedView(self: *Builder, mode: AtomicRMWMode, view: Value, index: []const Value, value: Value, opts: RedViewOpts) Value { + return self.emit(cuda_tile.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ + .scope = opts.scope, + .token = innerOpt(opts.token), + }, self.loc())); + } + + /// 13.3: `num_elem` elements of `dtype`, as a rank-0 pointer tile. + pub fn alloca(self: *Builder, dtype: DType, num_elem: i64, opts: AllocaOpts) Value { + return self.emit(cuda_tile.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); + } + + pub fn global(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, value: anytype, opts: GlobalOpts) void { + const ty = self.tileTy(shape, dtype); + const attr = cuda_tile.splatAttr(ty, self.elemAttr(dtype, value)); + self.appendGlobal(cuda_tile.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + /// A module-level dense buffer from a slice of Zig scalars (see `dense` + /// for the byte-count rule). + pub fn globalDense(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, values: anytype, opts: GlobalOpts) void { + checkDenseBytes("Builder.globalDense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + self.appendGlobal(cuda_tile.global(self.ctx, name, cuda_tile.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + fn appendGlobal(self: *Builder, op: *mlir.Operation) void { + const body = self.ct_body orelse @panic("Builder.global called before declareArgs"); + const first = body.firstOperation() orelse { + body.appendOwnedOperation(op); + return; + }; + body.insertOwnedOperationBefore(first, op); + } + + pub fn getGlobal(self: *Builder, name: []const u8, dtype: DType) Value { + return self.emit(cuda_tile.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); + } + + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addfOpts(lhs, rhs, .{}); + } + pub fn addfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subfOpts(lhs, rhs, .{}); + } + pub fn subfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.mulfOpts(lhs, rhs, .{}); + } + pub fn mulfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.divfOpts(lhs, rhs, .{}); + } + pub fn divfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.negf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.absf(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.fmaOpts(a, b, acc, .{}); + } + pub fn fmaOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn maxf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.maxfOpts(lhs, rhs, .{}); + } + pub fn maxfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(cuda_tile.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn minf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.minfOpts(lhs, rhs, .{}); + } + pub fn minfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(cuda_tile.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn pow(self: *Builder, x: Value, exponent: Value) Value { + return self.emit(cuda_tile.pow(self.ctx, x.inner, exponent.inner, self.loc())); + } + pub fn atan2(self: *Builder, x: Value, y: Value) Value { + return self.emit(cuda_tile.atan2(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{}, self.loc())); + } + /// 13.3: `exp` with an explicit rounding mode. + pub fn expRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.exp2Opts(x, false); + } + pub fn exp2Opts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(cuda_tile.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.sqrtOpts(x, .{}); + } + pub fn sqrtOpts(self: *Builder, x: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.rsqrtOpts(x, false); + } + pub fn rsqrtOpts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(cuda_tile.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.tan(self.ctx, x.inner, self.loc())); + } + pub fn sinh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.sinh(self.ctx, x.inner, self.loc())); + } + pub fn cosh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.cosh(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{}, self.loc())); + } + /// 13.2: `tanh` with an explicit rounding mode. + pub fn tanhRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.ceil(self.ctx, x.inner, self.loc())); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.floor(self.ctx, x.inner, self.loc())); + } + + pub fn cmpf(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value) Value { + return self.cmpfOpts(predicate, lhs, rhs, .ordered); + } + pub fn cmpfOpts(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, ordering: ComparisonOrdering) Value { + return self.emit(cuda_tile.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); + } + + pub fn mmaf(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.mmafOpts(a, b, acc, .{}); + } + pub fn mmafOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: MmaOpts) Value { + return self.emit(cuda_tile.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); + } + /// 13.3: block-scaled MMA. Scales are plain tiles in logical layout — + /// `a_scale` is `[M, K/V]`, `b_scale` is `[K/V, N]` — never pre-swizzled. + pub fn mmafScaled(self: *Builder, a: Value, b: Value, acc: Value, a_scale: Value, b_scale: Value) Value { + return self.emit(cuda_tile.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); + } + pub fn mmai(self: *Builder, a: Value, b: Value, acc: Value, a_signedness: Signedness, b_signedness: Signedness) Value { + return self.emit(cuda_tile.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); + } + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addiOpts(lhs, rhs, .{}); + } + pub fn addiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subiOpts(lhs, rhs, .{}); + } + pub fn subiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.muliOpts(lhs, rhs, .{}); + } + pub fn muliOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn mulhii(self: *Builder, x: Value, y: Value) Value { + return self.emit(cuda_tile.mulhii(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn divi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.diviOpts(lhs, rhs, .{ .signedness = signedness }); + } + pub fn diviOpts(self: *Builder, lhs: Value, rhs: Value, opts: DivIOpts) Value { + return self.emit(cuda_tile.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, .{ .rounding = opts.rounding }, self.loc())); + } + pub fn remi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn maxi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn mini(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn negi(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.negi(self.ctx, x.inner, .none, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.absi(self.ctx, x.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); + } + pub fn shri(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.maxf(l, r) else self.maxi(l, r, .signed); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.minf(l, r) else self.mini(l, r, .signed); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(cuda_tile.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + pub fn where(self: *Builder, cond: Value, x: anytype, y: anytype) Value { + var xv, var yv = self.coerce(x, y); + var cv = cond; + cv, xv = self.coerceShapes(cv, xv); + xv, yv = self.coerceShapes(xv, yv); + cv, _ = self.coerceShapes(cv, xv); + return self.select(cv, xv, yv); + } + + pub fn bitCast(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(cuda_tile.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { + return self.emit(cuda_tile.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(cuda_tile.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); + } + pub fn ftof(self: *Builder, src: Value, dtype: DType) Value { + return self.ftofOpts(src, dtype, .{}); + } + pub fn ftofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + const rounding: RoundingMode = opts.rounding orelse (if (dtype == .f8e8m0fnu) .zero else .nearest_even); + return self.emit(cuda_tile.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .{ .rounding = rounding }, self.loc())); + } + /// Float -> integer, truncating toward zero (the only rounding `ftoi` + /// admits). + pub fn ftoi(self: *Builder, src: Value, dtype: DType) Value { + return self.ftoiOpts(src, dtype, .{}); + } + pub fn ftoiOpts(self: *Builder, src: Value, dtype: DType, opts: FToIOpts) Value { + return self.emit(cuda_tile.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + /// Integer -> float. `f8e8m0fnu` is not a legal target; go through `f32`. + pub fn itof(self: *Builder, src: Value, dtype: DType) Value { + return self.itofOpts(src, dtype, .{}); + } + pub fn itofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (dtype == .f8e8m0fnu) @panic("Builder.itof: the dialect refuses integer -> f8e8m0fnu; convert to .f32 first (Builder.cast does)"); + return self.emit(cuda_tile.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn intToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(cuda_tile.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + pub fn ptrToInt(self: *Builder, src: Value) Value { + return self.emit(cuda_tile.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); + } + pub fn ptrToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(cuda_tile.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + /// 13.3: a sub-byte tile packed into `tile`. + pub fn pack(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); + } + /// 13.3: an i8 tile unpacked into `tile` of a sub-byte `dtype`. + pub fn unpack(self: *Builder, src: Value, shape: []const i64, dtype: DType) Value { + return self.emit(cuda_tile.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch: ftof / ftoi / itof / exti / trunci / + /// bitcast by element kind and width. Integers are signed; i1 is not. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) return self.ftof(src, dtype); + if (cur_is_float) return self.ftoi(src, dtype); + if (tgt_is_float) { + const signedness: Signedness = if (cur_dtype == .i1) .unsigned else .signed; + if (dtype == .f8e8m0fnu) return self.ftof(self.itofOpts(src, .f32, .{ .signedness = signedness }), dtype); + return self.itofOpts(src, dtype, .{ .signedness = signedness }); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) return self.exti(src, dtype, if (cur_dtype == .i1) .unsigned else .signed); + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.bitCast(src, dtype); + } + + pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + fn tileTyOf(self: *const Builder, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + if (!dtypes.tileLegal(shape)) std.debug.panic("Builder: {any} is not a legal tile shape", .{shape}); + return cuda_tile.tileType(self.ctx, shape, elem); + } + + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn expandDims(self: *Builder, src: Value, axis: usize) Value { + const in_shape = src.shape(); + var out: Value.Shape = .empty; + for (0..in_shape.len + 1) |i| { + if (i == axis) out.appendAssumeCapacity(1) else out.appendAssumeCapacity(in_shape.get(if (i < axis) i else i - 1)); + } + return self.reshape(src, out.constSlice()); + } + + pub fn cat(self: *Builder, lhs: Value, rhs: Value, dim: usize) Value { + var out = lhs.shape(); + out.set(dim, out.get(dim) + rhs.dim(dim)); + return self.emit(cuda_tile.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); + } + + pub fn extract(self: *Builder, src: Value, indices: []const Value, shape: []const i64) Value { + return self.emit(cuda_tile.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out: Value.Shape = .empty; + for (order) |i| out.appendAssumeCapacity(src_shape.get(@intCast(i))); + return self.emit(cuda_tile.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); + } + + pub fn transpose(self: *Builder, src: Value) Value { + std.debug.assert(src.rank() == 2); + return self.permute(src, &.{ 1, 0 }); + } + + pub fn offset(self: *Builder, ptrs: Value, off: Value) Value { + return self.emit(cuda_tile.offset(self.ctx, ptrs.inner, off.inner, self.loc())); + } + + fn reducedShape(src: Value, axis: usize) Value.Shape { + if (axis >= src.rank()) std.debug.panic("Builder.reduce: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + var out: Value.Shape = .empty; + const s = src.shape(); + for (0..s.len) |i| if (i != axis) out.appendAssumeCapacity(s.get(i)); + return out; + } + + fn checkAxis(src: Value, axis: usize) void { + if (axis >= src.rank()) std.debug.panic("Builder.scan: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + } + + fn combineBody(self: *Builder, ctx: anytype, elem: *const mlir.Type, combine: anytype) *mlir.Block { + const scalar = cuda_tile.tileType(self.ctx, &.{}, elem); + const body = mlir.Block.init(&.{ scalar, scalar }, &.{ self.loc(), self.loc() }); + self.pushBlock(body); + const element: Value = .{ .inner = body.argument(0), .kernel = self }; + const acc: Value = .{ .inner = body.argument(1), .kernel = self }; + const combined: Value = combine(self, element, acc, ctx); + _ = cuda_tile.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const result_ty = self.tileTyOf(reducedShape(args.src, args.axis).constSlice(), args.src.elemType()); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(cuda_tile.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); + } + + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + checkAxis(args.src, args.axis); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(cuda_tile.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); + } + + /// The `2N`-argument body of a variadic `reduce`/`scan`: arguments come in + /// (element, accumulator) pairs, one pair per source. + fn combineBodyMulti(self: *Builder, ctx: anytype, srcs: []const Value, combine: anytype) *mlir.Block { + const scratch = self.arena.allocator(); + const n = srcs.len; + const types = scratch.alloc(*const mlir.Type, 2 * n) catch @panic("OOM"); + const locs = scratch.alloc(*const mlir.Location, 2 * n) catch @panic("OOM"); + for (srcs, 0..) |s, i| { + const scalar = cuda_tile.tileType(self.ctx, &.{}, s.elemType()); + types[2 * i] = scalar; + types[2 * i + 1] = scalar; + locs[2 * i] = self.loc(); + locs[2 * i + 1] = self.loc(); + } + const body = mlir.Block.init(types, locs); + self.pushBlock(body); + const elements = scratch.alloc(Value, n) catch @panic("OOM"); + const accs = scratch.alloc(Value, n) catch @panic("OOM"); + for (0..n) |i| { + elements[i] = .{ .inner = body.argument(2 * i), .kernel = self }; + accs[i] = .{ .inner = body.argument(2 * i + 1), .kernel = self }; + } + const out: []const Value = combine(self, elements, accs, ctx); + if (out.len != n) std.debug.panic("Builder.reduceMulti: combine returned {d} values for {d} sources", .{ out.len, n }); + _ = cuda_tile.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduceMulti(self: *Builder, ctx: anytype, args: ReduceMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| result_types[i] = self.tileTyOf(reducedShape(s, args.axis).constSlice(), s.elemType()); + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(cuda_tile.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); + } + + pub fn scanMulti(self: *Builder, ctx: anytype, args: ScanMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| { + checkAxis(s, args.axis); + result_types[i] = s.type_(); + } + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(cuda_tile.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); + } + + const ReduceKind = enum { sum, max, min, prod }; + + fn combineKind(k: *Builder, element: Value, acc: Value, kind: ReduceKind) Value { + const is_float = element.isFloatElem(); + return switch (kind) { + .sum => if (is_float) k.addf(element, acc) else k.addi(element, acc), + .prod => if (is_float) k.mulf(element, acc) else k.muli(element, acc), + .max => if (is_float) k.maxf(element, acc) else k.maxi(element, acc, .signed), + .min => if (is_float) k.minf(element, acc) else k.mini(element, acc, .signed), + }; + } + + fn kindIdentity(self: *Builder, src: Value, kind: ReduceKind) *const mlir.Attribute { + const dt = self.mlirElemToDType(src.elemType()); + if (isFloatDtype(dt)) { + return self.identityAttr(dt, switch (kind) { + .sum => @as(f64, 0), + .prod => @as(f64, 1), + .max => -std.math.inf(f64), + .min => std.math.inf(f64), + }); + } + const bits: u32 = intBitwidth(dt); + const min_int: i64 = if (bits == 64) std.math.minInt(i64) else -(@as(i64, 1) << @intCast(bits - 1)); + const max_int: i64 = if (bits == 64) std.math.maxInt(i64) else (@as(i64, 1) << @intCast(bits - 1)) - 1; + return self.identityAttr(dt, switch (kind) { + .sum => @as(i64, 0), + .prod => @as(i64, 1), + .max => min_int, + .min => max_int, + }); + } + + fn reduceKind(self: *Builder, src: Value, axis: usize, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.reduce({}, .{ .src = src, .axis = axis, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn sum(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .sum); + } + pub fn max(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .max); + } + pub fn min(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .min); + } + pub fn prod(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .prod); + } + + fn scanKind(self: *Builder, src: Value, axis: usize, opts: ScanOpts, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.scan({}, .{ .src = src, .axis = axis, .reverse = opts.reverse, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn cumsum(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .sum); + } + pub fn cumsumOpts(self: *Builder, src: Value, axis: usize, opts: ScanOpts) Value { + return self.scanKind(src, axis, opts, .sum); + } + pub fn cumprod(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .prod); + } + pub fn cummax(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .max); + } + pub fn cummin(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .min); + } + + /// `for iv in [lower, upper) step` carrying `inits`; close with + /// `scope.yield(.{...})` (`scope.close()` without inits), then read + /// `scope.results`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `cuda_tile.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openFor: inits"); + + // Literals take the type of the first bound that is a Value, else + // `i32`; all three must agree (`AllTypesMatch` on the op). + const lb_v, const ub_v, const step_v = common.forBounds(self, Value, lower, upper, step, i32Bound); + + var block_types: [N + 1]*const mlir.Type = undefined; + block_types[0] = lb_v.type_(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); + block_types[i + 1] = v.type_(); + inits_inner[i] = v.inner; + } + const block_locs: [N + 1]*const mlir.Location = @splat(self.loc()); + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + .opts = opts, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = common.resultTypes(self, Value, result_types, "openIfElse: result_types"), + }; + } + + /// An unstructured loop carrying `inits` (`scope.close()` ends a body that + /// carries nothing, in place of `scope.yield`); leave with `scope.breakWith` + /// inside an `if`, and close the body with `scope.yield`. + pub fn openLoop(self: *Builder, inits: anytype) LoopScope(common.arity(@TypeOf(inits), "openLoop: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openLoop: inits"); + var block_types: [N]*const mlir.Type = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); + block_types[i] = v.type_(); + inits_inner[i] = v.inner; + } + const block_locs: [N]*const mlir.Location = @splat(self.loc()); + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .inits_inner = inits_inner, + .result_types = block_types, + .carried = carried, + }; + } + + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + _ = cuda_tile.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// Append `return`, verify the module, and serialize it to a NUL-terminated + /// string owned by the kernel's allocator. `results` must be empty: an + /// `entry` returns nothing; outputs reach the host through pointers. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + std.debug.assert(results.len == 0); + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = cuda_tile.return_(self.ctx, &.{}, self.loc()).appendTo(current); + } + + try common.verify(self.module, "dsl/cuda_tile"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation()}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, ir: [:0]const u8) !void { + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "empty entry round-trips" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "empty"); + defer b.deinit(); + _ = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.find(u8, ir, "assume") != null); + try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.find(u8, ir, "return") != null); + try expectRoundTrip(ctx, ir); +} + +test "add_one over pointer tiles matches the XLA milestone kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "add_one"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .in = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + .out = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + }, .{}); + + const offsets = b.iota(128, .i32); + const loaded = b.loadPtrOpts(a.in.offset(offsets), .{}); + const sum = loaded.tile.add(1.0); + _ = b.storePtrOpts(a.out.offset(offsets), sum, .{ .token = loaded.token }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.find(u8, ir, "offset") != null); + try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); + try expectRoundTrip(ctx, ir); +} + +test "vector_add over partition views" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .y = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + const shape = [_]i64{ 8192, 128 }; + const strides = [_]i64{ 128, 1 }; + const tile = [_]i64{ 64, 64 }; + const vx = b.partitionView(b.tensorView(a.x, &shape, &strides), &tile, .{}); + const vy = b.partitionView(b.tensorView(a.y, &shape, &strides), &tile, .{}); + const vo = b.partitionView(b.tensorView(a.out, &shape, &strides), &tile, .{}); + + const bid = b.tileBlockId(); + const idx = [_]Value{ bid.x, bid.y }; + const xt = b.load(vx, &idx); + const yt = b.load(vy, &idx); + _ = b.store(xt.add(yt), vo, &idx); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); + try expectRoundTrip(ctx, ir); +} + +test "gemm with a for loop and mmaf" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "gemm"); + defer b.deinit(); + const a = try b.declareArgs(.{ .a = .{ .ptr = .bf16 }, .bm = .{ .ptr = .bf16 }, .c = .{ .ptr = .f32 } }); + + const m: i64 = 1024; + const n: i64 = 1024; + const k: i64 = 512; + const pa = b.partitionView(b.tensorView(a.a, &.{ m, k }, &.{ k, 1 }), &.{ 128, 64 }, .{}); + const pb = b.partitionView(b.tensorView(a.bm, &.{ k, n }, &.{ n, 1 }), &.{ 64, 128 }, .{}); + const pc = b.partitionView(b.tensorView(a.c, &.{ m, n }, &.{ n, 1 }), &.{ 128, 128 }, .{}); + + const bid = b.tileBlockId(); + const acc0 = b.zeros(&.{ 128, 128 }, .f32); + + var loop = b.openFor(0, @divExact(k, 64), 1, .{acc0}); + { + const kt = loop.iv; + const acc = loop.carried[0]; + const at = b.load(pa, &.{ bid.x, kt }); + const bt = b.load(pb, &.{ kt, bid.y }); + loop.yield(.{b.mmaf(at, bt, acc)}); + } + _ = b.store(loop.results[0], pc, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "for ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); + try expectRoundTrip(ctx, ir); +} + +test "if, if-else, loop with break, reduce, scan, cast, select" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "control"); + defer b.deinit(); + // A scalar reaches the kernel through a rank-0 pointer. + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .n = .{ .ptr = .i32 } }); + const n = b.loadPtr(a.n); + try std.testing.expect(n.isScalar()); + + const v = b.partitionView(b.tensorView(a.x, &.{ 256, 64 }, &.{ 64, 1 }), &.{ 8, 64 }, .{}); + const bid = b.tileBlockId(); + const t = b.load(v, &.{ bid.x, bid.y }); + + // Reductions and scans over the tile. + const row_sum = b.sum(t, 1); + const col_max = b.max(t, 0); + const running = b.cumsum(t, 1); + try std.testing.expectEqual(@as(usize, 1), row_sum.rank()); + try std.testing.expectEqual(@as(i64, 8), row_sum.dim(0)); + try std.testing.expectEqual(@as(i64, 64), col_max.dim(0)); + try std.testing.expectEqual(@as(usize, 2), running.rank()); + + // A variadic reduce: argmax over axis 1 as (max, index). + const cols = b.broadcastTo(b.reshape(b.iota(64, .i32), &.{ 1, 64 }), &.{ 8, 64 }); + const argmax = b.reduceMulti({}, .{ + .srcs = &.{ t, cols }, + .axis = 1, + .identities = &.{ b.identityAttr(.f32, -std.math.inf(f64)), b.identityAttr(.i32, 0) }, + .combine = struct { + fn c(k: *Builder, elements: []const Value, accs: []const Value, _: void) []const Value { + const better = elements[0].gt(accs[0]); + return k.yield(.{ k.select(better, elements[0], accs[0]), k.select(better, elements[1], accs[1]) }); + } + }.c, + }); + try std.testing.expectEqual(@as(usize, 2), argmax.len); + try std.testing.expectEqual(@as(i64, 8), argmax[1].dim(0)); + + // Casts and select. + const as_i32 = t.to(.i32); + const back = as_i32.to(.f32); + const picked = b.where(t.gt(0.0), back, t.neg()); + + // if / if-else carrying a value. + const is_first = bid.x.eq(0); + var only = b.openIf(is_first); + { + _ = b.store(picked, v, &.{ bid.x, bid.y }); + only.close(); + } + + var branch = b.openIfElse(is_first, .{b.tileTy(&.{ 8, 64 }, .f32)}); + { + branch.yieldThen(.{picked}); + } + { + branch.yieldElse(.{t}); + } + const chosen = branch.results[0]; + + // loop: count up to n, break when reached. + var lp = b.openLoop(.{b.constant(.i32, 0)}); + { + const i = lp.carried[0]; + var done = b.openIf(i.ge(n)); + { + lp.breakWith(.{i}); + done.close(); + } + lp.yield(.{i.add(1)}); + } + _ = lp.results[0]; + + // a loop carrying nothing ends with close(). + var once = b.openLoop(.{}); + { + var first = b.openIf(is_first); + { + once.breakWith(.{}); + first.close(); + } + once.close(); + } + + // for with an early `continue` inside an `if`, and i64 bounds from a Value. + var skip = b.openFor(b.constant(.i64, 0), 16, 1, .{b.constant(.i64, 0)}); + { + const acc = skip.carried[0]; + var odd = b.openIf(skip.iv.bitAnd(1).eq(1)); + { + skip.continueWith(.{acc}); + odd.close(); + } + skip.yield(.{acc.add(skip.iv)}); + } + try std.testing.expect(skip.results[0].isScalar()); + + _ = b.store(chosen, v, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.find(u8, ir, "itof") != null); + try std.testing.expect(std.mem.find(u8, ir, "select") != null); + try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.find(u8, ir, "break ") != null); + try std.testing.expect(std.mem.find(u8, ir, "yield") != null); + try expectRoundTrip(ctx, ir); +} + +test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "misc"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .cnt = .{ .ptr = .i32 }, .rows = .{ .ptr = .i32 } }); + b.global("lut", &.{16}, .f32, 1.5, .{ .constant = true }); + + const rows = b.loadPtr(a.rows); + const tv = b.tensorViewDyn(a.x, &.{ .{ .dynamic = rows }, .{ .static = 64 } }, &.{ .{ .static = 64 }, .{ .static = 1 } }); + const extents = b.tensorShape(tv); + try std.testing.expectEqual(@as(usize, 2), extents.len); + const v = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = .nan }); + const tiles = b.indexSpaceShape(v); + try std.testing.expectEqual(@as(usize, 2), tiles.len); + const sv = b.stridedView(tv, &.{ 8, 64 }, &.{ 2, 1 }, .{}); + _ = b.load(sv, &.{ b.constant(.i32, 0), b.constant(.i32, 0) }); + + const bid = b.tileBlockId(); + const loaded = b.loadOpts(v, &.{ bid.x, bid.y }, .{ .hints = &.{.{ .arch = .sm_120, .allow_tma = true }} }); + const tok = b.print("tile %f\n", &.{loaded.tile}); + + const counter = a.cnt.offset(b.iota(8, .i32)); + const old = b.atomicRmw(.add, counter, b.full(&.{8}, 1, .i32), .{ .token = tok }); + const cas = b.atomicCas(counter, old.tile, b.full(&.{8}, 0, .i32), .{ .token = old.token }); + const joined = b.joinTokens(&.{ loaded.token, cas.token, b.makeToken() }); + // Atomic reductions refuse a padded view; the default options must verify. + const unpadded = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = null }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{ .token = joined }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{}); + + // A gather/scatter view: the sparse index is a rank-1 tile of row ids. + const gsv = b.gatherScatterView(tv, &.{ 8, 64 }, 0, .zero); + const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); + try std.testing.expectEqual(@as(i64, 8), gathered.dim(0)); + try std.testing.expectEqual(@as(i64, 64), gathered.dim(1)); + _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); + + const lut = b.getGlobal("lut", .f32); + const scratch = b.alloca(.f32, 64, .{}); + _ = b.loadPtr(lut.offset(b.iota(16, .i32))); + _ = b.storePtr(scratch.offset(b.iota(64, .i32)), b.zeros(&.{64}, .f32)); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "global @lut") != null); + try std.testing.expect(std.mem.find(u8, ir, "get_global @lut") != null); + try std.testing.expect(std.mem.find(u8, ir, "tensor_view") != null); + try expectRoundTrip(ctx, ir); +} diff --git a/dsl/cuda_tile/control_flow.zig b/dsl/cuda_tile/control_flow.zig new file mode 100644 index 00000000..26979e0a --- /dev/null +++ b/dsl/cuda_tile/control_flow.zig @@ -0,0 +1,185 @@ +//! `cuda_tile` scope wrappers, like `dsl/common/control_flow.zig` but with +//! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an +//! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. + +const common = @import("dsl/common/builder"); +const std = @import("std"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); + +fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, comptime mnemonic: enum { yield, continue_, break_ }, values: anytype, comptime what: []const u8) void { + if (block.terminator() != null) return; + const buf = common.innerValues(ValueT, N, values, what); + const op = switch (mnemonic) { + .yield => cuda_tile.yield(kernel.ctx, &buf, kernel.loc()), + .continue_ => cuda_tile.continue_(kernel.ctx, &buf, kernel.loc()), + .break_ => cuda_tile.break_(kernel.ctx, &buf, kernel.loc()), + }; + _ = op.appendTo(block); +} + +pub const ForOpts = struct { + /// Compare the induction variable unsigned. + unsigned_cmp: bool = false, +}; + +/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it, or +/// `close()` when it carries nothing. +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + opts: ForOpts = .{}, + + const Self = @This(); + + /// `continue` in the current block, inside a zero-result `if`: skips + /// the rest of the iteration and terminates that branch. + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "ForScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); + k.popBlock(); + const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.opts.unsigned_cmp, k.loc()); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn close(self: *Self) void { + const k = self.kernel; + terminate(ValueT, 0, k, self.then_block, .yield, .{}, "IfOnlyScope.close"); + k.popBlock(); + _ = cuda_tile.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()).appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.then_block, .yield, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + fn endElse(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = cuda_tile.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +/// `loop iter_values(...)`: the body ends with `continue` (`yield`, or +/// `close()` when it carries nothing) and leaves through `break` (`breakWith`, +/// inside an `if`). `results` are the broken. +pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + inits_inner: [N]*const mlir.Value, + result_types: [N]*const mlir.Type, + carried: [N]ValueT, + results: [N]ValueT = undefined, + + const Self = @This(); + + /// `break` in the current block; terminates the enclosing `if` branch. + pub fn breakWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .break_, values, "LoopScope.breakWith"); + } + + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "LoopScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("LoopScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("LoopScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); + k.popBlock(); + const loop_op = cuda_tile.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); + _ = loop_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = loop_op.result(i), .kernel = k }; + } + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/cuda_tile/dtype.zig b/dsl/cuda_tile/dtype.zig new file mode 100644 index 00000000..a9193839 --- /dev/null +++ b/dsl/cuda_tile/dtype.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Every element type `cuda_tile.tile` accepts (Types.td: +/// CudaTile_TileElementType). `i4`, `f4e2m1fn` need 13.3; `f8e8m0fnu` needs +/// 13.2. `i4` is a tile element only — a pointer cannot point at it. +pub const DType = enum { + i1, + i4, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + tf32, + f64, + f8e4m3fn, + f8e5m2, + f8e8m0fnu, + f4e2m1fn, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .tf32 => .float(ctx, .tf32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), + .f4e2m1fn => .float(ctx, .f4e2m1fn), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .tf32, .f64, .f8e4m3fn, .f8e5m2, .f8e8m0fnu, .f4e2m1fn => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i4, .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32, .tf32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i4 => 4, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |f_name| { + const dt = @field(DType, f_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized CUDA Tile IR DType"); +} + +/// SharedVerifiers.h verifyTileSize: every dimension a power of two and the +/// tile at most 2^24 elements. Checked in Zig so the failure names the shape. +pub fn tileLegal(shape: []const i64) bool { + var elems: i64 = 1; + for (shape) |d| { + if (d <= 0 or (d & (d - 1)) != 0) return false; + elems *= d; + } + return elems <= (1 << 24); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "tileLegal" { + try std.testing.expect(tileLegal(&.{})); + try std.testing.expect(tileLegal(&.{ 64, 128 })); + try std.testing.expect(!tileLegal(&.{3})); + try std.testing.expect(!tileLegal(&.{ 4096, 4096, 2 })); +} diff --git a/dsl/cute/BUILD.bazel b/dsl/cute/BUILD.bazel new file mode 100644 index 00000000..ef5fc6d9 --- /dev/null +++ b/dsl/cute/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/cute/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/cuda", + "//mlir/dialects/cute_ir", + "//mlir/dialects/cute_nvgpu", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/nvvm", + "//mlir/dialects/scf", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/cute/README.md b/dsl/cute/README.md new file mode 100644 index 00000000..946c542d --- /dev/null +++ b/dsl/cute/README.md @@ -0,0 +1,186 @@ +# CuTe DSL kernels in Zig + +`dsl/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the +Python `cutlass.cute` API: a kernel takes tensors, indexes them with +coordinates, and reads `thread_idx()` and friends from `cute.arch`. It is the +fourth instance of the `dsl/triton` / `dsl/mosaic_tpu` / +`dsl/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent scalar arithmetic, and a +`zml.kernel.cute.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. + +The dialect bindings live in `mlir/dialects/cute_ir` (`cute`), +`mlir/dialects/cute_nvgpu`, and `mlir/dialects/nvvm` and `mlir/dialects/cuda` +for NVIDIA's `nvvm` and `cuda` (operation builders under `.ops`), with types +and attributes through each dialect's C API. + +## A kernel + +The Python notebook's `naive_elementwise_add_kernel`, in Zig: + +```zig +const zml = @import("zml"); +const cute = zml.kernel.cute; + +const Cfg = struct { m: i64, n: i64 }; + +pub const NaiveElementwiseAdd = cute.Kernel(Cfg, .{ + .name = "naive_elementwise_add_kernel", + .inputs = &.{ "gA", "gB" }, + .outputs = &.{"gC"}, + .run = run, +}); + +fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + // A `tensor` argument is XLA's device pointer composed with a static + // layout; the Python kernel takes `cute.Tensor` the same way. + const t = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gB = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gC = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + + const n = t.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + + // gC[mi, ni] = gA[mi, ni] + gB[mi, ni] + t.gC.set(.{ mi, ni }, t.gA.get(.{ mi, ni }).add(t.gB.get(.{ mi, ni }))); +} + +// In a model (`kernel.launch(grid=..., block=...)`): +const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, .{ + .cfg = .{ .m = 64, .n = 128 }, + .grid = .{ 32, 1, 1 }, + .block = .{ 256, 1, 1 }, +}).gC; +``` + +## Python to Zig + +Put each control-flow region in a Zig `{ ... }` block immediately after +`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call (or +`close()` for a scope without values) inside the block. For `openIfElse`, use +separate blocks ending in `yieldThen` and `yieldElse` (`elseBranch()` and +`close()` when it has no results); likewise, give `openWhile` separate +`yieldBefore` and `yieldAfter` blocks. Keep the scope variable outside these +blocks so its `.results` remain accessible afterward. The braces make the +generated control flow visible and keep region-local values from escaping +their scope; the builder's `open` and `yield` calls still create and close +the actual IR regions. + +| `cutlass.cute` | `dsl/cute` | +|--------------------------------------------|-------------------------------------------------| +| `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | +| `gA[mi, ni]` / `gC[mi, ni] = v` | `gA.get(.{ mi, ni })` / `gC.set(.{ mi, ni }, v)`| +| `cute.make_layout(shape, stride=...)` | `b.makeLayout(shape, .{ .stride = ... })` | +| `cute.make_tensor(ptr, layout)` | `b.makeTensor(ptr, layout)` | +| `cute.arch.thread_idx()` → `(x, y, z)` | `b.threadIdx().x` | +| `block_idx()`, `block_dim()`, `grid_dim()` | `blockIdx()`, `blockDim()`, `gridDim()` | +| `cute.arch.alloc_smem(T, n, align)` | `b.allocSmem(.f32, n, align)` | +| `cute.arch.sync_threads()` | `b.syncThreads()` | +| `cute.arch.shuffle_sync_bfly(v, lane)` | `b.shuffleXor(v, lane)` | +| `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | +| `x.to(Float32)` | `x.to(.f32)` | +| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaAtom(...)` | +| `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | +| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, .{})` | +| `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | +| `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | +| `cute.gemm(...)` | `b.gemm(...)` | +| `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | +| `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | + +* **Layouts accept static and dynamic extents.** `makeLayout` takes Zig slices + for a tracked static `Layout`, or nested tuples of integers and SSA `Value`s + for a `View`. Dynamic shape and stride leaves retain their integer widths. + The tracked `Tensor` keeps its shape and stride in Zig; use `makeTensorView` + with dynamic layouts. Integers stay static in coordinate types. +* **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's + compact left-most stride like `cute.make_layout`; a `tensor` argument + defaults to XLA's row-major layout, which is what the buffer holds. +* **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; + `constant(dtype, v)` picks another type and `coerce` lifts literals to the other + operand's type. +* **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's + `nvvm` and `cuda` dialects (`nvvm.ops`, `cuda.ops`, from the + `mlir/dialects/nvvm` and `mlir/dialects/cuda` packages) and upstream's + `llvm`, so every operation is + verified as it is built; nothing is left unregistered. +* **The module is just the kernel.** `finish` prints a module holding only + `func.func @name(...)`; the CuTe compiler makes every public function a + kernel entry, so there is no `gpu.module` or host launch function to write. +* **Generic form.** `finish` and `finishProgram` print the generic operation + form: NVIDIA's compiler parses with its own MLIR revision, whose custom + assembly differs from ours (`llvm.load`/`llvm.store` alignment, for one). +* **Explicit result types:** Zig covers local tiling, TMA partitioning, tiled + copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives + their layouts in the compiler, so callers spell out the resulting CuTe type + with `View` instead of relying on Python's dynamic type objects. +* **TMA tilers follow Python.** `makeTiledTmaAtom` accepts a shape tuple, tile, + or layout, composes it with the GMEM identity layout, and removes the SMEM + stage mode when present. The lower-level `makeTiledTmaLoadAtom` and + `makeTiledTmaStoreAtom` take an already composed coordinate map. `copy` + makes store atoms executable automatically; loads require `tma_bar_ptr`. +* **Typed result types.** Operations whose result type CuTe derives + (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled + with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, + `smemDescViewType` or `tiledCopyType`; nothing is parsed from text. +* **Architecture sections.** `Builder` groups TMA (`cute.nvgpu.cpasync`, + SM90+) and tcgen05 / tensor memory (`cute.nvgpu.tcgen05`, SM100) in their + own sections; mbarriers, fences and bulk-copy groups sit with `cute.arch`. + Each doc comment names its minimum architecture when it is above SM80. + +## Programs with a host launch + +Kernels that need launch-time CuTe objects (TMA descriptors, tiled MMA atoms, +scheduler parameters) use `zml.kernel.cute.Program` instead of `Kernel`. Its +`run` callback emits one `gpu.module` with the `cuda.kernel`s +(`b.beginFunction(name, .cuda_kernel)`), then a public host `func.func` +(`b.beginFunction(name, .host)`) that builds those objects from the XLA +buffers and calls `b.launchEx`. The custom call carries no grid or block: +XLA runs the host function with the real buffers and launches (or records in +a command buffer) whatever it launched. `CallOpts.scalars` are passed to the +host function after the buffers. `zml/kernels/moe/cute/persistent_mxfp4.zig` +is the reference: a warp-specialized SM100 block-scaled GEMM whose host +function builds four TMA tensor maps and a PDL launch. + +## The custom call + +`ops.cute` emits `__gpu$xla.gpu.cute` with a printed dictionary: `name`, +`kernel_type = "cute"`, `ir` (the textual module), `grid` and `block` +(3-element arrays), optional `zeroed_outputs`. XLA compiles it in process +(`lib/libcute_compiler.so`, which pulls in `lib/cutlass_ir.so`, both from the +sandbox pointed at by `--xla_gpu_cuda_data_dir`), takes the kernel's +dynamic shared memory requirement from the compiler's metadata, and launches +the cubin over `grid` × `block`. Every operand and result is one raw device +pointer in the default layout, operands first; the count is not checked +against the kernel signature, so `inputs`/`outputs` must match `declareArgs`. +A module that carries its own `gpu.module` and host launch still works; +it then owns the launch configuration and `grid`/`block` must be left out. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cute.mlir`, `.cute.json` and `.cubin`. + +## Finding what exists + +```sh +# the DSL: Value's fluent methods (add, mul, lt, to, exp, ...) +awk '/^pub const Value = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' + +# the DSL: everything on Tensor and Builder +awk '/^pub const Tensor = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' +awk '/^pub const Builder = struct/,0' dsl/cute/builder.zig | grep ' pub fn ' + +# every bound op of the dialect +grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_nvgpu/cute_nvgpu.zig +``` + +`grep -n '^test "' dsl/cute/builder.zig` lists the tests; each prints a +kernel that round-trips through the verifier. diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig new file mode 100644 index 00000000..8f791f80 --- /dev/null +++ b/dsl/cute/builder.zig @@ -0,0 +1,4953 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const cuda = @import("mlir/dialects/cuda"); +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const nvvm = @import("mlir/dialects/nvvm"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// `CUtensorMapDataType`. +pub const TensorMapDataType = enum(u32) { + uint8 = 0, + uint16 = 1, + uint32 = 2, + int32 = 3, + uint64 = 4, + int64 = 5, + float16 = 6, + float32 = 7, + float64 = 8, + bfloat16 = 9, + float32_ftz = 10, + tfloat32 = 11, + tfloat32_ftz = 12, + @"16u4_align8b" = 13, + @"16u4_align16b" = 14, + @"16u6_align16b" = 15, +}; + +/// `CUtensorMapSwizzle`. +pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; +/// `CUtensorMapL2promotion`. +pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; +/// `CUtensorMapInterleave`. +pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; +/// `CUtensorMapFloatOOBfill`. +pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; + +pub const TensorMapSpec = struct { + data_type: TensorMapDataType, + rank: u32, + global_dims: [5]u64 = @splat(1), + /// Byte strides of dimensions 1..rank-1. + global_strides: [4]u64 = @splat(0), + box_dims: [5]u32 = @splat(1), + element_strides: [5]u32 = @splat(1), + interleave: TensorMapInterleave = .none, + swizzle: TensorMapSwizzle, + l2_promotion: TensorMapL2Promotion = .none, + oob_fill: TensorMapOobFill = .none, +}; + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); + std.testing.refAllDecls(Tensor); + std.testing.refAllDecls(View); + std.testing.refAllDecls(Atom); +} + +/// `nvvm` and `cuda` are NVIDIA's (`nvvm.ops`, `cuda.ops`, handles `cute_nvvm` +/// and `cuda`); `llvm` is upstream's, whose types and operations NVIDIA's +/// compiler reads as its own. +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, gpu, cute_ir, cute_nvgpu, nvvm, cuda, llvm, arith, scf, math, cf, vector }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const MAX_RANK = 8; +pub const Dims = stdx.BoundedArray(i64, MAX_RANK); + +pub const MemorySpace = cute_ir.MemorySpace; + +/// A CuTe algebra expression assembled from Zig values. Kernel authors use +/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; the DSL builds +/// the algebra attribute of the kind each use needs. +pub const AlgebraExpr = union(enum) { + static: i64, + /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. + dynamic: i64, + /// A dynamic integer with an explicit bit width and divisibility. + dynamic_int: struct { width: u8, divisible_by: i64 = 1 }, + all, + basis: ScaledBasis, + tuple: []const AlgebraExpr, +}; + +pub const AlgebraToken = enum { + /// CuTe's slice placeholder (`_`). + all, + /// A dynamic algebra leaf (`?`). + dynamic, +}; + +pub const ScaledBasis = struct { + numerator: i64, + denominator: i64 = 1, + modes: [MAX_RANK]u8 = @splat(0), + mode_count: u8, +}; + +/// A dynamic CuTe integer whose runtime value is known to be divisible by a +/// static amount. CuTe carries this fact in dependent coordinate-tensor types +/// after tiling and slicing, for example `?{div=128}`. +pub const ConstrainedDynamic = struct { + divisible_by: u64, +}; + +pub const LayoutSpec = struct { + shape: AlgebraExpr, + stride: AlgebraExpr, +}; + +/// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn dtype(self: Value) DType { + return dtypes.mlirToDType(self.kern().ctx, self.type_()); + } + + pub fn isFloat(self: Value) bool { + const t = self.type_(); + inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; + } + return false; + } + + pub fn isInt(self: Value) bool { + return self.type_().isA(mlir.IntegerType) != null; + } + + pub fn isPtr(self: Value) bool { + return self.type_().isA(cute_ir.PtrType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Signed division on integers (Python `//` on Int32). + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.remf(k.ctx, l.inner, r.inner, k.loc()) else arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloat()) @panic("Value.cdiv is integer-only"); + return k.emit(arith.ceildivsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + if (self.isFloat()) return k.emit(arith.negf(k.ctx, self.inner, k.loc())); + return k.constant(self.dtype(), 0).sub(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return k.emit(if (self.isFloat()) math.absf(k.ctx, self.inner, k.loc()) else math.absi(k.ctx, self.inner, k.loc())); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.minimumf(k.ctx, l.inner, r.inner, k.loc()) else arith.minsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike + /// the NaN-propagating `maximum`. + pub fn maxnum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shrLogical(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitCast(self: Value, dt: DType) Value { + const k = self.kern(); + if (dtypeBitwidth(self.dtype()) != dtypeBitwidth(dt)) @panic("Value.bitCast requires equal bit widths"); + return k.emit(arith.bitcast(k.ctx, self.inner, dt.toMlir(k.ctx), k.loc())); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .slt, .olt); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sle, .ole); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sgt, .ogt); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sge, .oge); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .eq, .oeq); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .ne, .une); + } + + pub fn exp(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp(k.ctx, self.inner, k.loc())); + } + + pub fn log(self: Value) Value { + const k = self.kern(); + return k.emit(math.log(k.ctx, self.inner, k.loc())); + } + + pub fn log2(self: Value) Value { + const k = self.kern(); + return k.emit(math.log2(k.ctx, self.inner, k.loc())); + } + + pub fn exp2(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp2(k.ctx, self.inner, k.loc())); + } + + pub fn ceil(self: Value) Value { + const k = self.kern(); + if (!self.isFloat()) @panic("Value.ceil is float-only"); + return k.emit(math.ceil(k.ctx, self.inner, k.loc())); + } + + pub fn sqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.sqrt(k.ctx, self.inner, k.loc())); + } + + pub fn rsqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.rsqrt(k.ctx, self.inner, k.loc())); + } + + pub fn tanh(self: Value) Value { + const k = self.kern(); + return k.emit(math.tanh(k.ctx, self.inner, k.loc())); + } + + /// Python `x.to(T)`: int <-> int is signed, int <-> float goes through + /// `sitofp`/`fptosi`, same-width floats through `convertf`. + pub fn to(self: Value, dt: DType) Value { + const k = self.kern(); + const src = self.dtype(); + if (src == dt) return self; + const ty = dt.toMlir(k.ctx); + const src_f = isFloatDtype(src); + const dst_f = isFloatDtype(dt); + const sw = dtypeBitwidth(src); + const dw = dtypeBitwidth(dt); + if (src_f and dst_f) { + if (sw == dw) return k.emit(arith.convertf(k.ctx, self.inner, ty, .{}, k.loc())); + return k.emit(if (dw > sw) arith.extf(k.ctx, self.inner, ty, k.loc()) else arith.truncf(k.ctx, self.inner, ty, k.loc())); + } + if (src_f) return k.emit(arith.fptosi(k.ctx, self.inner, ty, k.loc())); + if (dst_f) return k.emit(arith.sitofp(k.ctx, self.inner, ty, k.loc())); + if (sw == dw) return self; + if (src == .i1) return k.emit(arith.extui(k.ctx, self.inner, ty, k.loc())); + return k.emit(if (dw > sw) arith.extsi(k.ctx, self.inner, ty, k.loc()) else arith.trunci(k.ctx, self.inner, ty, k.loc())); + } +}; + +/// `cute.Layout`: a static shape and stride, carried in Zig so coordinates +/// and element counts need no MLIR queries. +pub const Layout = struct { + inner: *const mlir.Value, + shape: Dims, + stride: Dims, + + pub fn rank(self: Layout) usize { + return self.shape.len; + } + + pub fn size(self: Layout) i64 { + var n: i64 = 1; + for (self.shape.constSlice()) |d| n *= d; + return n; + } + + pub fn dim(self: Layout, i: usize) i64 { + return self.shape.get(i); + } +}; + +/// `cute.Tensor`: an engine (pointer) composed with a layout. `get`/`set` +/// are the Python `t[coord]` / `t[coord] = v`. +pub const Tensor = struct { + inner: *const mlir.Value, + kernel: *Builder, + layout: Layout, + dtype: DType, + space: MemorySpace, + + pub fn shape(self: Tensor) []const i64 { + return self.layout.shape.constSlice(); + } + + pub fn rank(self: Tensor) usize { + return self.layout.rank(); + } + + pub fn size(self: Tensor) i64 { + return self.layout.size(); + } + + pub fn dim(self: Tensor, i: usize) i64 { + return self.layout.dim(i); + } + + /// `t[coord]`: `coord` is a tuple of `Value`s and ints, one per mode. + pub fn get(self: Tensor, coord: anytype) Value { + const k = self.kernel; + const c = k.makeCoord(coord); + return k.emit(cute_ir.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); + } + + /// `t[coord] = value`; the value is converted to the element type. + pub fn set(self: Tensor, coord: anytype, value_: anytype) void { + const k = self.kernel; + const c = k.makeCoord(coord); + const v = k.constant(self.dtype, value_).to(self.dtype); + _ = cute_ir.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); + } + + /// Erase Zig's tracked layout while retaining the MLIR memref. CuTe + /// layout algebra derives the layouts returned by tiling/partitioning; + /// those values are represented by `View` because Zig cannot recompute + /// their dependent types. + pub fn view(self: Tensor) View { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn value(self: Tensor) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } +}; + +/// A CuTe tensor or pointer with a layout derived by the CuTe compiler. +pub const View = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: View) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: View) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn get(self: View, coord: anytype, result_dtype: DType) Value { + const c = self.kernel.makeCoord(coord); + return self.kernel.emit(cute_ir.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); + } + + pub fn set(self: View, coord: anytype, value_: Value) void { + self.kernel.emitVoid(cute_ir.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); + } +}; + +/// A copy/MMA atom or a tiled atom. Atom fields are immutable SSA values; +/// `setField` returns the atom carrying the updated runtime field. +pub const Atom = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: Atom) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: Atom) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn setField(self: Atom, field: []const u8, value_: anytype) Atom { + const T = @TypeOf(value_); + const new_value = if (T == Value or T == Tensor or T == View or T == Atom or T == Layout) + self.kernel.asValue(value_) + else + self.kernel.lift(value_); + const result = self.kernel.emit(cute_nvgpu.atom_set_value( + self.kernel.ctx, + self.inner, + new_value.inner, + self.type_(), + .string(self.kernel.ctx, field), + self.kernel.loc(), + )); + return .{ .inner = result.inner, .kernel = self.kernel }; + } +}; + +pub const TmaDescriptor = struct { + atom: Atom, + tensor: View, +}; + +pub const TmaPartition = struct { + smem: View, + targets: []const View, +}; + +/// Element encoding of a TMA transfer when it differs from the tensor dtype. +pub const TmaFormat = enum { + default, + /// Packed 4-bit elements, two per byte in shared memory (NVFP4 operands + /// of the `mxf4nvf4` MMA). + u4, + /// 4-bit elements expanded to one per byte in shared memory (FP4 operands + /// of the `mxf8f6f4` MMA). + u4_unpack_u8, + u16, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM90. +pub const CacheHintSm90 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM100. +pub const CacheHintSm100 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +pub const CtaGroup = enum(u8) { + one = 1, + two = 2, + + fn toNvvm(self: CtaGroup) nvvm.CtaGroup { + return switch (self) { + .one => .cta_1, + .two => .cta_2, + }; + } +}; +pub const MBarrierScope = nvvm.MBarrierScope; +pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; + +/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. +pub fn ctaCount(group: CtaGroup) u8 { + return @backingInt(group); +} + +pub const Tcgen05SmemSwizzle = enum(u3) { + none = 0, + swizzle_128b_atom_32b = 1, + swizzle_128b = 2, + swizzle_64b = 4, + swizzle_32b = 6, +}; + +/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. +pub const Tcgen05SmemDesc = struct { + pub const Options = struct { + leading_byte_offset: u32 = 0, + stride_byte_offset: u32 = 0, + version: u2 = 1, + base_offset: u3 = 0, + layout: Tcgen05SmemSwizzle = .none, + }; + + pub fn build(b: *Builder, start_address: Value, opts: Options) Value { + std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); + std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); + const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | + (@as(u64, opts.stride_byte_offset >> 4) << 32) | + (@as(u64, opts.version) << 46) | + (@as(u64, opts.base_offset) << 49) | + (@as(u64, @backingInt(opts.layout)) << 61); + const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); + } +}; + +/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. +pub const Tcgen05InstrDesc = struct { + pub const Options = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType = .f32, + m_dim: u16, + n_dim: u16, + a_major: OperandMajorMode = .k, + b_major: OperandMajorMode = .k, + a_negate: bool = false, + b_negate: bool = false, + }; + + pub fn build(comptime opts: Options) u32 { + comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); + comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); + const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); + const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); + const valid = (half_inputs and opts.c_dtype == .f32) or + (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or + (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or + (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or + (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); + if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); + const c: u32 = switch (opts.c_dtype) { + .f16 => 0, + .f32 => 1, + .i32 => 2, + else => @compileError("unsupported tcgen05 accumulator dtype"), + }; + return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | + (@as(u32, @intFromBool(opts.a_negate)) << 13) | + (@as(u32, @intFromBool(opts.b_negate)) << 14) | + (@as(u32, @backingInt(opts.a_major)) << 15) | + (@as(u32, @backingInt(opts.b_major)) << 16) | + (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); + } + + fn inputFormat(comptime dtype: DType) u32 { + return switch (dtype) { + .f16, .f8e4m3fn => 0, + .bf16, .f8e5m2, .i8 => 1, + .f32 => 2, + .f4e2m1fn => 5, + else => @compileError("unsupported tcgen05 input dtype"), + }; + } +}; + +/// Typed description of a non-multicast tiled TMA atom: the element type, +/// bits moved per copy, basis of one box, and coordinate layout of the tensor. +pub const TmaConfig = struct { + dtype: DType, + copy_bits: u32, + global_basis: LayoutSpec, + coordinate_layout: LayoutSpec, + coordinate_rank: u8, + format: TmaFormat = .default, + /// Load completion group: with `.two`, each CTA's load completes on + /// the mbarrier of the pair's even CTA. Ignored by TMA stores. + cta_group: CtaGroup = .one, + mode: enum { tiled, gather4 } = .tiled, +}; + +pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensorTileS2GOp = struct {}; + +pub const BlockScaledMma = struct { + atom: Atom, + tiled: Atom, +}; + +/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. +/// The instruction `kind` fixes B, the scales and K. +pub const BlockScaledMmaConfig = struct { + kind: Kind = .mxfp4_mxfp8, + /// M is 128 for one CTA, or 128 or 256 for a CTA pair. + m: u16 = 128, + n: u16, + cta_group: CtaGroup = .one, + + pub const Kind = enum { + /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 + /// elements, K = 32 (PTX `kind::mxf8f6f4`). + mxfp4_mxfp8, + /// NVFP4 x NVFP4: E4M3 scales every 16 elements, K = 64 (PTX + /// `kind::mxf4nvf4.block16`). + nvfp4_nvfp4, + }; + + pub fn k(self: BlockScaledMmaConfig) u16 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 64, + }; + } + + fn bType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e4m3fn, + .nvfp4_nvfp4 => .f4e2m1fn, + }; + } + + fn scaleType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e8m0fnu, + .nvfp4_nvfp4 => .f8e4m3fn, + }; + } + + fn vecSize(self: BlockScaledMmaConfig) u8 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 16, + }; + } + + fn validate(self: BlockScaledMmaConfig) void { + const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); + const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; + if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); + } + } +}; + +fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { + return switch (config.cta_group) { + .one => .sm_90, + .two => .sm_100_2sm, + }; +} + +fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { + return switch (format) { + .u4 => .U4, + .u4_unpack_u8 => .U4_UNPACK_U8, + .u16 => .U16, + .default => switch (dtype) { + .f8e4m3fn => .U8, + .f32 => .F32_RN, + else => @panic("TMA format must be specified for this dtype"), + }, + }; +} + +fn tmaFormatAttribute(b: *Builder, config: TmaConfig) ?*const mlir.Attribute { + if (config.format == .default) return null; + const attr = cute_nvgpu.TmaDataFormatAttr.get(b.ctx, .{ .value = tmaDataFormat(config.format, config.dtype) }) catch @panic("invalid TMA format"); + return attr.attribute(); +} + +pub const Index3 = struct { x: Value, y: Value, z: Value }; + +pub const LaunchConfig = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn value(self: LaunchConfig) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } +}; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument. A `tensor` + /// argument is that pointer composed with a static layout at the top of + /// the kernel, the way the Python DSL passes `cute.Tensor` arguments. + pub const Kind = union(enum) { + ptr: DType, + tensor: TensorSpec, + /// A type constructed through a dialect/type API. + mlir_type: MlirTypeSpec, + }; + + pub const MlirTypeSpec = struct { + type_: *const mlir.Type, + grid_constant: bool = false, + cuda_grid_constant: bool = false, + }; + + pub const TensorSpec = struct { + dtype: DType, + shape: []const i64, + /// Defaults to XLA's row-major layout. + stride: ?[]const i64 = null, + alignment: u64 = 16, + }; +}; + +pub const LayoutOpts = struct { + /// Defaults to CuTe's compact left-most stride, like `cute.make_layout`. + stride: ?[]const i64 = null, +}; + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +pub const Builder = struct { + pub const FunctionKind = enum { func_only_kernel, cuda_kernel, host }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + gpu_body: ?*mlir.Block = null, + name: []const u8, + function_kind: FunctionKind = .func_only_kernel, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + args: []const ArgSpec = &.{}, + tensors: []?Tensor = &.{}, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + const module: *mlir.Module = .init(.unknown(ctx)); + errdefer module.deinit(); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + }; + } + + /// Open a complete CuTe program. Unlike `open`, which emits the compact + /// func.func-only ABI consumed directly by XLA, a program owns a + /// `gpu.module` for one or more `cuda.kernel` operations and can append a + /// host `func.func` that constructs CuTe launch objects and calls + /// `cuda.launch_ex`. + pub fn openProgram(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + var self = try open(allocator, ctx, name); + self.module.operation().setAttribute(gpu.containerModule(ctx)); + const body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, "kernels", body, self.loc()); + _ = gpu_module.appendTo(self.module.body()); + self.gpu_body = body; + self.function_kind = .cuda_kernel; + return self; + } + + /// Start another function in a complete program after `endFunction`. + pub fn beginFunction(self: *Builder, name: []const u8, kind: FunctionKind) void { + std.debug.assert(self.entry_block == null and self.func_op == null); + self.name = name; + self.function_kind = kind; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// One field per kernel argument: `.{ .a = .{ .tensor = .{ .dtype = .f32, .shape = &.{n} } } }` + /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".field_names; + + const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const inner = @field(raw, variant); + arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + .ptr => .{ .ptr = inner }, + .tensor => .{ .tensor = .{ + .dtype = inner.dtype, + .shape = inner.shape, + .stride = if (@hasField(@TypeOf(inner), "stride")) inner.stride else null, + .alignment = if (@hasField(@TypeOf(inner), "alignment")) inner.alignment else 16, + } }, + .mlir_type => .{ .mlir_type = .{ + .type_ = inner.type_, + .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, + } }, + } }; + } + try self.declareArgsLow(arg_specs); + + var named: ArgsOf(Spec) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { + Tensor => self.tensors[i].?, + else => self.arg(i), + }; + } + return named; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".field_names; + comptime var names: [in.len][]const u8 = undefined; + comptime var types: [in.len]type = undefined; + inline for (in, 0..) |f, i| { + names[i] = f; + types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; + } + return @Struct(.auto, null, &names, &types, &@splat(.{})); + } + + pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *l, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt, .gmem, 16), + .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), + .mlir_type => |typed| typed.type_, + } catch return error.InvalidMlir; + l.* = self.loc(); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; + } + + const func_op = switch (self.function_kind) { + .func_only_kernel => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{}, + .location = self.loc(), + .verify = false, + .extra_attributes = &.{ + cute_ir.kernel(ctx), + gpu.kernel(ctx), + }, + }), + .cuda_kernel => cuda.kernel(ctx, .{ + .name = self.name, + .function_type = .function(ctx, arg_types, &.{}), + .body = entry, + .arg_attrs = .array(ctx, arg_attrs), + .extra_attributes = &.{ + cute_ir.kernel(ctx), + gpu.kernel(ctx), + }, + .location = self.loc(), + }), + .host => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{.int(ctx, .i32)}, + .location = self.loc(), + .verify = false, + .extra_attributes = &.{llvm.emitCInterface(ctx)}, + }), + }; + _ = func_op.appendTo(if (self.function_kind == .cuda_kernel) + self.gpu_body orelse @panic("cuda_kernel requires Builder.openProgram") + else + self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + self.args = args; + + self.tensors = try scratch.alloc(?Tensor, args.len); + for (self.tensors, args, 0..) |*t, a, i| { + t.* = switch (a.kind) { + .ptr => null, + .tensor => |spec| self.makeTensor(self.arg(i), self.makeLayout(spec.shape, .{ + .stride = spec.stride orelse self.rowMajor(spec.shape), + })), + .mlir_type => null, + }; + } + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + /// Append an operation which does not return an SSA value. + pub fn emitVoid(self: *Builder, op: *mlir.Operation) void { + std.debug.assert(op.numResults() == 0); + _ = op.appendTo(self.currentBlock()); + } + + /// Append a multi-result operation and return all of its SSA values. + /// + /// Most of the small CuTe kernels only need one-result operations, but + /// TMA descriptor construction and TMA partitioning return an atom and a + /// coordinate tensor together. Keeping both results from one operation + /// is required: rebuilding either value changes the descriptor identity. + pub fn emitResults(self: *Builder, comptime N: usize, op: *mlir.Operation) [N]Value { + std.debug.assert(op.numResults() == N); + _ = op.appendTo(self.currentBlock()); + var result: [N]Value = undefined; + for (&result, 0..) |*value, i| value.* = .{ .inner = op.result(i), .kernel = self }; + return result; + } + + /// Convert any DSL handle into a scalar wrapper without changing its + /// MLIR type. This keeps the low-level generated bindings usable while + /// callers work with `Tensor`, `View`, and `Atom` handles. + pub fn asValue(self: *Builder, value_: anytype) Value { + return switch (@TypeOf(value_)) { + Value => value_, + Tensor => value_.value(), + View => value_.value(), + Atom => value_.value(), + LaunchConfig => value_.value(), + Layout => .{ .inner = value_.inner, .kernel = self }, + else => @compileError("expected a CuTe DSL handle, got " ++ @typeName(@TypeOf(value_))), + }; + } + + fn innerSlice(self: *Builder, values: anytype) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("OOM"); + switch (@typeInfo(@TypeOf(values))) { + .@"struct" => { + inline for (values, 0..) |value, i| out[i] = self.asValue(value).inner; + }, + else => { + for (values, out) |value, *dst| dst.* = self.asValue(value).inner; + }, + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// `#cuda.dev_max_shared_memory_optin`, the device's opt-in maximum of + /// shared memory per block, for a kernel's `cu_attrs`. + pub fn devMaxSharedMemoryOptin(self: *Builder) *const mlir.Attribute { + return (cuda.ops.DevMaxSharedMemoryOptinAttr.get(self.ctx, .{}) catch @panic("invalid CUDA attribute")).attribute(); + } + + pub fn setFunctionAttribute(self: *Builder, attribute: mlir.NamedAttribute) void { + const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); + function.setAttribute(attribute); + } + + // ==================== cuda host launch ==================== + + /// `@kernels::@`, a kernel of the nested GPU module. + fn kernelSymbol(self: *Builder, kernel_name: []const u8) *const mlir.Attribute { + return .symbolRef(self.ctx, "kernels", &.{.init(self.ctx, kernel_name)}); + } + + pub fn kernelSmemSize(self: *Builder, kernel_name: []const u8) Value { + return self.emit(cute_ir.kernel_smem_size( + self.ctx, + .int(self.ctx, .i64), + self.kernelSymbol(kernel_name), + self.loc(), + )); + } + + /// Convert the stream placeholder used by a generated host entry into a + /// CUDA stream. The native CuTe host bridge substitutes XLA's current + /// stream when it executes the wrapper. + pub fn cudaStream(self: *Builder) Value { + return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.constant(.i64, 0).inner, self.loc())); + } + + pub const LaunchOptions = struct { + grid: [3]Value, + block: [3]Value, + dynamic_smem: Value, + stream: Value, + cluster: ?[3]Value = null, + cooperative: bool = false, + use_pdl: bool = false, + }; + + /// Build the mutable CUDA launch configuration used by `cuda.launch_ex`. + pub fn makeLaunchConfig(self: *Builder, opts: LaunchOptions) LaunchConfig { + const config = self.emit(cuda.launch_cfg_create(self.ctx, .{ + .max_attrs = 17, + .block = .{ opts.block[0].inner, opts.block[1].inner, opts.block[2].inner }, + .dynamic_smem = opts.dynamic_smem.inner, + .grid = .{ opts.grid[0].inner, opts.grid[1].inner, opts.grid[2].inner }, + .stream = opts.stream.inner, + .location = self.loc(), + })); + if (opts.cluster) |cluster| { + self.emitVoid(cuda.launch_cfg_cluster_dim(self.ctx, config.inner, .{ cluster[0].inner, cluster[1].inner, cluster[2].inner }, self.loc())); + } + self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); + self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); + return .{ .inner = config.inner, .kernel = self }; + } + + /// Request a CUDA kernel launch from a generated host function. The host + /// bridge records this request; XLA performs or command-buffer-records the + /// actual launch on its stream. + pub fn launchEx(self: *Builder, kernel_name: []const u8, config: LaunchConfig, arguments: anytype) Value { + const values = self.innerSlice(arguments); + return self.emit(cuda.launch_ex(self.ctx, .{ + .config = config.inner, + .inputs = values, + .callee = self.kernelSymbol(kernel_name), + .assume_kernel_attr = cuda.assumeKernelAttr(self.ctx, true), + .location = self.loc(), + })); + } + + /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. + pub fn tensorMapType(self: *Builder) *const mlir.Type { + return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); + } + + /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / + /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. + fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { + const ctx = self.ctx; + const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); + const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); + const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, + .location = self.loc(), + })); + for (values, 0..) |v, i| { + const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = &.{array.inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, + .location = self.loc(), + })); + const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); + self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); + } + return array; + } + + /// Encode a tiled map in a host function and return both the map and CUDA status. + /// The caller supplies the final device address and handles the returned status. + pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { + const ctx = self.ctx; + // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. + ctx.setAllowUnregisteredDialects(true); + const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); + const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( + ctx, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, + self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, + addr.inner, + self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, + self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, + cuda.resultType(ctx), + self.tensorMapType(), + self.loc(), + )); + return .{ .map = results[1], .result = results[0] }; + } + + /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` + /// (`&tensor_map` for a `__grid_constant__` parameter). + pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { + const ctx = self.ctx; + const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); + return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{llvm_ptr.inner} }, + .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, + .location = self.loc(), + })); + } + + pub fn cudaResultStatus(self: *Builder, result: Value) Value { + return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); + } + + pub fn returnHostStatus(self: *Builder, status: Value) void { + self.emitVoid(func.returns(self.ctx, &.{status.inner}, self.loc())); + } + + /// Finish the current function but retain the surrounding module so a + /// host launch function can be emitted next. + pub fn endFunction(self: *Builder, block: ?[3]i32) void { + const current = self.currentBlock(); + if (current.terminator() == null) { + const terminator = if (self.function_kind == .cuda_kernel) + cuda.return_(self.ctx, &.{}, self.loc()) + else + func.returns(self.ctx, &.{}, self.loc()); + _ = terminator.appendTo(current); + } + if (block) |dims| { + const function = self.func_op orelse @panic("endFunction before declareArgs"); + function.setAttribute(nvvm.reqntid(self.ctx, dims)); + } + self.func_op = null; + self.entry_block = null; + self.block_stack.clear(); + self.args = &.{}; + self.tensors = &.{}; + } + + pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { + std.debug.assert(self.entry_block == null and self.func_op == null); + try common.verify(self.module, "dsl/cute"); + return self.renderModule(); + } + + fn renderModule(self: *Builder) FinishError![:0]const u8 { + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + // Always the generic form: NVIDIA's compiler parses the module with its + // own MLIR revision, whose custom assembly differs from ours (e.g. + // `cute.make_composed_layout`, or `llvm.load`/`llvm.store` printing + // their alignment as a `` property). + try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); + } + + // ==================== types ==================== + + pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { + return cute_ir.pointerType(self.ctx, space, alignment, .{ .element_type = dt.toMlir(self.ctx) }); + } + + pub fn swizzledPtrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) !*const mlir.Type { + return (try cute_ir.PtrType.get(self.ctx, .{ + .valueType = dt.toMlir(self.ctx), + .addressSpace = space, + .alignment = alignment, + .swizzle = (try self.swizzleAttr(swizzle)).attribute(), + })).type_(); + } + + fn swizzleAttr(self: *Builder, swizzle: Swizzle) !*const cute_ir.SwizzleAttr { + return cute_ir.SwizzleAttr.get(self.ctx, .{ + .bits = std.math.cast(u32, swizzle.bits) orelse return error.InvalidMlir, + .base = std.math.cast(u32, swizzle.base) orelse return error.InvalidMlir, + .shift = std.math.cast(i32, swizzle.shift) orelse return error.InvalidMlir, + }); + } + + pub fn intToPtr(self: *Builder, address: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const ptr_type = self.ptrTy(dt, space, alignment) catch @panic("invalid CuTe pointer type"); + return self.emit(cute_ir.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); + } + + /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared + /// memory, `i64` for global memory), for hand-written PTX operands. + pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { + const ptr = self.asValue(pointer); + return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); + } + + /// Reinterpret a pointer without changing its address. This is the typed + /// counterpart of Python CuTe's `recast_ptr`; packed MXFP4 buffers use it + /// to expose their byte storage to a scalar correctness implementation. + pub fn recastPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const result_type = self.ptrTy(dt, space, alignment) catch @panic("invalid recast pointer type"); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + pub fn recastSwizzledPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) Value { + const result_type = self.swizzledPtrTy(dt, space, alignment, swizzle) catch @panic("invalid swizzled pointer type"); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + /// The scalar of a single extent, otherwise the tuple of `dims`. + fn dimsExpr(self: *Builder, dims: []const i64) AlgebraExpr { + if (dims.len == 1) return .{ .static = dims[0] }; + const elements = self.arena.allocator().alloc(AlgebraExpr, dims.len) catch @panic("OOM"); + for (dims, elements) |d, *e| e.* = .{ .static = d }; + return .{ .tuple = elements }; + } + + /// Build a CuTe algebra expression from integers, nested Zig tuples, + /// `AlgebraToken`s, `ConstrainedDynamic`s and `ScaledBasis` values. + pub fn expr(self: *Builder, value: anytype) AlgebraExpr { + const T = @TypeOf(value); + if (T == AlgebraExpr) return value; + if (T == AlgebraToken) return switch (value) { + .all => .all, + .dynamic => .{ .dynamic = 1 }, + }; + if (T == ScaledBasis) return .{ .basis = value }; + if (T == ConstrainedDynamic) { + if (value.divisible_by == 0) @panic("CuTe constrained dynamic divisor must be positive"); + return .{ .dynamic = @intCast(value.divisible_by) }; + } + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .static = @intCast(value) }, + .@"struct" => |info| { + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.expr(@field(value, field)); + return .{ .tuple = elements }; + }, + else => @compileError("unsupported CuTe algebra value " ++ @typeName(T)), + } + } + + /// The algebra attribute `A` (`cute_ir.IntTupleAttr`, `ShapeAttr`, ...) + /// of `e`. + fn algebraAttr(self: *Builder, comptime A: type, e: AlgebraExpr) *const A { + const result = switch (e) { + .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), + .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), + .basis => |b| if (@hasDecl(A, "getBasis")) blk: { + const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); + for (modes, b.modes[0..b.mode_count]) |*mode, m| mode.* = m; + break :blk A.getBasis(self.ctx, b.numerator, b.denominator, modes); + } else std.debug.panic("{s} has no scaled-basis leaf", .{@typeName(A)}), + .tuple => |elements| blk: { + const attrs = self.arena.allocator().alloc(*const A, elements.len) catch @panic("OOM"); + for (elements, attrs) |element, *attr| attr.* = self.algebraAttr(A, element); + break :blk A.getTuple(self.ctx, attrs); + }, + }; + return result catch std.debug.panic("invalid CuTe algebra for {s}", .{@typeName(A)}); + } + + fn layoutAttr(self: *Builder, spec: LayoutSpec) *const cute_ir.LayoutAttr { + return cute_ir.LayoutAttr.get(self.ctx, .{ + .shape = self.algebraAttr(cute_ir.ShapeAttr, spec.shape), + .stride = self.algebraAttr(cute_ir.StrideAttr, spec.stride), + }) catch @panic("invalid CuTe layout"); + } + + fn intTupleType(self: *Builder, e: AlgebraExpr) *const mlir.Type { + return (cute_ir.IntTupleType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.IntTupleAttr, e).attribute() }) catch @panic("invalid CuTe integer tuple")).type_(); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first like + /// Python's `ScaledBasis(value, modes)`. CuTe prints it innermost first: + /// `basis(1, 2, .{ 0, 1 })` is `1/2@1@0`. + pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { + _ = self; + if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); + const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; + if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); + var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); + return result; + } + + pub fn layoutSpec(self: *Builder, shape: anytype, stride: anytype) LayoutSpec { + return .{ .shape = self.expr(shape), .stride = self.expr(stride) }; + } + + pub fn layoutType(self: *Builder, spec: LayoutSpec) *const mlir.Type { + return (cute_ir.LayoutType.get(self.ctx, .{ .attr = self.layoutAttr(spec).attribute() }) catch @panic("invalid CuTe layout")).type_(); + } + + pub fn composedLayoutType(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) *const mlir.Type { + const attr = cute_ir.ComposedLayoutAttr.get(self.ctx, .{ + .inner = .{ .swizzle = self.swizzleAttr(swizzle) catch @panic("invalid CuTe swizzle") }, + .offset = self.algebraAttr(cute_ir.IntTupleAttr, self.expr(offset)), + .outer = self.layoutAttr(outer), + }) catch @panic("invalid CuTe composed layout"); + return (cute_ir.ComposedLayoutType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe composed layout")).type_(); + } + + pub fn memrefType(self: *Builder, dtype: DType, space: MemorySpace, alignment: u64, layout_type: *const mlir.Type) *const mlir.Type { + const pointer = self.ptrTy(dtype, space, alignment) catch @panic("invalid CuTe pointer type"); + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + pub fn smemDescType(self: *Builder) *const mlir.Type { + return (cute_nvgpu.SmemDescType.get(self.ctx, .{}) catch @panic("invalid shared-memory descriptor")).type_(); + } + + /// `!cute_nvgpu.smem_desc_view` of a shared-memory descriptor over `layout`. + pub fn smemDescViewType(self: *Builder, layout: LayoutSpec) *const mlir.Type { + return (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ + .desc = self.smemDescType(), + .layout = self.layoutAttr(layout).attribute(), + }) catch @panic("invalid shared-memory descriptor view")).type_(); + } + + /// `!cute.tiled_copy` of `copy_atom` over `layout_copy_tv`, tiled by the + /// modes of `tiler_mn`. + pub fn tiledCopyType(self: *Builder, copy_atom: *const mlir.Type, layout_copy_tv: LayoutSpec, tiler_mn: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TiledCopyType.get(self.ctx, .{ + .copyAtom = copy_atom, + .layoutCopyTv = self.layoutAttr(layout_copy_tv).attribute(), + .tilerMn = self.tileAttr(tiler_mn).attribute(), + }) catch @panic("invalid tiled copy")).type_(); + } + + pub fn memrefTypeFromPointer(self: *Builder, pointer_type: *const mlir.Type, layout_type: *const mlir.Type) *const mlir.Type { + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + /// `cute.static` of an integer tuple such as `128` or `.{ 0, 4 }`. + pub fn staticIntTuple(self: *Builder, value: anytype) View { + const result = self.emit(cute_ir.static(self.ctx, self.intTupleType(self.expr(value)), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { + const result = self.emit(cute_ir.static(self.ctx, self.layoutType(spec), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const Swizzle = struct { bits: i64, base: i64, shift: i64 }; + + pub fn staticSwizzle(self: *Builder, spec: Swizzle) View { + const attr = self.swizzleAttr(spec) catch @panic("invalid CuTe swizzle"); + const ty = (cute_ir.SwizzleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe swizzle")).type_(); + const result = self.emit(cute_ir.static(self.ctx, ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Construct `swizzle o offset o outer_layout` through the CuTe dialect. + pub fn makeComposedLayout(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) View { + const inner = self.staticSwizzle(swizzle); + const offset_value = self.emit(cute_ir.make_int_tuple(self.ctx, &.{}, self.intTupleType(self.expr(offset)), self.loc())); + const outer_value = self.staticLayout(outer); + const result_ty = self.composedLayoutType(swizzle, offset, outer); + const result = self.emit(cute_ir.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Compose a raw pointer and any CuTe layout value. The memref result type + /// is obtained through the dialect C API rather than parsed from assembly. + pub fn makeTensorView(self: *Builder, pointer: anytype, layout: anytype) View { + const p = self.asValue(pointer); + const l = self.asValue(layout); + const result_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, p.inner, result_ty, .{ .layout = l.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coordTensorType(self: *Builder, rank: usize, layout: LayoutSpec) *const mlir.Type { + if (rank == 0 or rank > MAX_RANK) @panic("coordTensorType rank must be in 1..MAX_RANK"); + const zeros: [MAX_RANK]i64 = @splat(0); + return self.coordTensorTypeWithTuple(self.dimsExpr(zeros[0..rank]), layout); + } + + /// Coordinate-tensor type with an explicit arithmetic-tuple expression, + /// such as `.{ 0, ConstrainedDynamic{ .divisible_by = 128 }, AlgebraToken.dynamic }`. + /// This represents the divisibility information produced by CuTe tiling + /// without requiring kernel code to spell textual MLIR types. + pub fn coordTensorTypeWithTuple(self: *Builder, tuple: anytype, layout: LayoutSpec) *const mlir.Type { + return (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = self.intTupleType(self.expr(tuple)), .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + } + + fn rowMajor(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + var i = shape.len; + while (i > 0) : (i -= 1) { + s[i - 1] = acc; + acc *= shape[i - 1]; + } + return s; + } + + fn leftMost(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + for (shape, 0..) |d, i| { + s[i] = acc; + acc *= d; + } + return s; + } + + // ==================== cute.core ==================== + + fn layoutResult(comptime Shape: type) type { + return switch (@typeInfo(Shape)) { + .pointer => Layout, + else => View, + }; + } + + /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static + /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. + pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { + if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ + .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, + }); + var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const shape_expr = self.coordExpr(&shape_values, shape); + const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { + var stride: AlgebraExpr = .{ .static = 1 }; + var stride_value: ?Value = null; + var index: usize = 0; + break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); + }; + const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); + const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); + const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); + const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { + if (shape == .tuple) { + const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); + for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); + return .{ .tuple = elements }; + } + if (shape == .static and shape.static == 1) return .{ .static = 0 }; + const result = stride.*; + if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); + if (shape == .static and stride.* == .static) { + stride.static *= shape.static; + } else { + const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { + const v: Value = .{ .inner = values[index.*], .kernel = self }; + index.* += 1; + break :blk v; + } else self.lift(shape.static); + stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); + stride.* = dynamicExpr(stride_value.*.?); + } + return result; + } + + fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); + const stride = opts.stride orelse self.leftMost(shape); + if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); + const layout_ty = self.layoutType(.{ .shape = self.dimsExpr(shape), .stride = self.dimsExpr(stride) }); + const l = self.emit(cute_ir.static(self.ctx, layout_ty, self.loc())); + return .{ .inner = l.inner, .shape = Dims.fromSlice(shape) catch unreachable, .stride = Dims.fromSlice(stride) catch unreachable }; + } + + /// `cute.make_tensor(ptr, layout)`. + pub fn makeTensor(self: *Builder, ptr: Value, layout: Layout) Tensor { + const pt = ptr.type_().isA(cute_ir.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); + const elem = pt.valueType() orelse @panic("makeTensor: untyped pointer"); + const memref_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); + const v = self.emit(cute_ir.make_view(self.ctx, ptr.inner, memref_ty, .{ .layout = layout.inner }, self.loc())); + return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.addressSpace() }; + } + + /// The algebra of a coordinate whose `Value` leaves are dynamic and + /// appended to `dyn`. A one-element tuple is its element. + fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { + const T = @TypeOf(coord); + if (T == Value) { + dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); + return dynamicExpr(coord); + } + if (T == AlgebraToken) return switch (coord) { + .all => .all, + .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), + }; + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .static = @intCast(coord) }, + .@"struct" => |info| { + if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); + if (info.field_names.len == 1) return self.coordExpr(dyn, @field(coord, info.field_names[0])); + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.coordExpr(dyn, @field(coord, field)); + return .{ .tuple = elements }; + }, + else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), + } + } + + fn dynamicExpr(value: Value) AlgebraExpr { + std.debug.assert(value.isInt()); + const width = dtypeBitwidth(value.dtype()); + return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; + } + + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` + /// placeholders. Integer leaves stay static in the dependent CuTe type. + pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { + if (@TypeOf(coord) == Value) return self.makeCoord(.{coord}); + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const attr = self.algebraAttr(cute_ir.CoordAttr, self.coordExpr(&dyn, coord)); + const ty = (cute_ir.CoordType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("bad coord")).type_(); + return self.emit(cute_ir.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; + } + + /// A dynamic/static CuTe integer tuple, used for pointer arithmetic in + /// tensor and shared memory. It accepts the same nested Zig values as a + /// coordinate, but preserves integer-tuple semantics. + pub fn makeIntTuple(self: *Builder, values: anytype) View { + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const ty = self.intTupleType(self.coordExpr(&dyn, values)); + const value = self.emit(cute_ir.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// A one-element integer tuple holding a value known to be a multiple of + /// `divisor`. Ops that derive alignment from an offset, such as + /// `cute.add_offset`, need that to keep the operand's alignment. + pub fn makeIntTupleDivBy(self: *Builder, value: anytype, comptime divisor: u32) View { + const constrained = self.assumeDivBy(value, divisor); + const tuple = self.emit(cute_ir.make_int_tuple(self.ctx, &.{constrained.inner}, self.intTupleType(.{ .dynamic = divisor }), self.loc())); + return .{ .inner = tuple.inner, .kernel = self }; + } + + /// Refine an integer SSA value with the divisibility information CuTe + /// derives while slicing a tiled coordinate tensor. + pub fn assumeDivBy(self: *Builder, value: anytype, comptime divisor: u32) Value { + if (divisor == 0) @compileError("assumeDivBy divisor must be non-zero"); + const constrained = cute_ir.constrainedIntType(self.ctx, .i32, divisor) catch @panic("invalid CuTe constrained integer"); + return self.emit(cute_ir.assume(self.ctx, self.lift(value).to(.i32).inner, constrained, self.loc())); + } + + /// Build a coordinate tensor from a runtime origin and a static layout. + /// This is the direct form of the `get_iter`/`make_arith_tuple_iter`/ + /// `make_view` sequence emitted by the Python CuTe front end after a tile + /// is selected from a larger coordinate tensor. + pub fn makeCoordTensor(self: *Builder, origin: anytype, layout: anytype) View { + const tuple = self.makeIntTuple(origin); + const layout_value = self.asValue(layout); + const iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); + const iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, iterator_type, .{ .value_ = tuple.inner }, self.loc())); + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a coordinate tensor from the arithmetic-tuple iterator of an + /// existing tensor. CuTe's Python front end emits this sequence after it + /// slices a TMA-partitioned coordinate tensor: + /// + /// get_iter -> deref_arith_tuple_iter -> get_leaves -> make_int_tuple + /// -> make_arith_tuple_iter -> make_view + /// + /// Keeping that sequence explicit matters for TMA. The iterator carries + /// the descriptor coordinates selected by `slice`; constructing a fresh + /// tuple from the scheduler indices would lose CuTe's coordinate algebra. + pub fn rebuildCoordTensorIterator( + self: *Builder, + comptime leaf_count: usize, + input: anytype, + tuple: anytype, + comptime leaf_divisibility: [leaf_count]u32, + layout: anytype, + ) View { + if (leaf_count == 0 or leaf_count > MAX_RANK) @panic("invalid coordinate tuple leaf count"); + const source = self.asValue(input); + const source_type = source.type_().isA(cute_ir.CoordTensorType) orelse + std.debug.panic("rebuildCoordTensorIterator: expected coordinate tensor, got {f}", .{source.type_()}); + const source_tuple_type = source_type.arithTuple(); + const source_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch + @panic("invalid source arithmetic tuple iterator")).type_(); + const source_iterator = self.emit(cute_ir.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); + const source_tuple = self.emit(cute_ir.deref_arith_tuple_iter( + self.ctx, + source_iterator.inner, + source_tuple_type, + self.loc(), + )); + + const leaf_types = self.arena.allocator().alloc(*const mlir.Type, leaf_count) catch @panic("OOM"); + // 0 marks a leaf the producer knows to be statically zero, 1 a leaf + // with no known divisibility. + inline for (leaf_divisibility, 0..) |divisor, i| { + leaf_types[i] = self.intTupleType(if (divisor == 0) .{ .static = 0 } else .{ .dynamic = divisor }); + } + const leaves = self.emitResults(leaf_count, cute_ir.get_leaves( + self.ctx, + source_tuple.inner, + leaf_types, + self.loc(), + )); + + const rebuilt_tuple_type = self.intTupleType(self.expr(tuple)); + const rebuilt_tuple = self.emit(cute_ir.make_int_tuple( + self.ctx, + self.innerSlice(leaves[0..]), + rebuilt_tuple_type, + self.loc(), + )); + const rebuilt_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch + @panic("invalid rebuilt arithmetic tuple iterator")).type_(); + const rebuilt_iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, rebuilt_iterator_type, .{ .value_ = rebuilt_tuple.inner }, self.loc())); + const layout_value = self.asValue(layout); + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ + .arithTuple = rebuilt_tuple_type, + .layout = layout_value.type_(), + }) catch @panic("invalid rebuilt coordinate tensor")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, rebuilt_iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `!cute.shape` of a CuTe algebra value such as `.{ .{ 128, 16 }, 1, 1, 2 }` + /// or `.{ AlgebraToken.dynamic, AlgebraToken.dynamic }`. + pub fn shapeType(self: *Builder, shape: anytype) *const mlir.Type { + return (cute_ir.ShapeType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.ShapeAttr, self.expr(shape)).attribute() }) catch @panic("invalid CuTe shape")).type_(); + } + + /// Static `cute.make_shape` of a CuTe algebra value. + pub fn makeShape(self: *Builder, shape: anytype) View { + const value = self.emit(cute_ir.make_shape(self.ctx, &.{}, self.shapeType(shape), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// Dynamic `cute.make_shape`. The result type describes which leaves are + /// runtime values (for example `!cute.shape<"(?,?,?)">`). Keeping these + /// leaves dynamic is significant to CuTe's TMA layout algebra when a + /// runtime extent happens to be one. + pub fn makeShapeValues(self: *Builder, values: anytype, result_type: *const mlir.Type) View { + const value = self.emit(cute_ir.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + pub fn tileToShape(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tile_to_shape( + self.ctx, + self.asValue(input).inner, + self.asValue(shape).inner, + order.inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Static `cute.make_tile` from mode extents. + pub fn makeTile(self: *Builder, shape: []const i64) View { + const modes = self.arena.allocator().alloc(LayoutSpec, shape.len) catch @panic("OOM"); + for (shape, modes) |extent, *mode| mode.* = .{ .shape = .{ .static = extent }, .stride = .{ .static = 1 } }; + const value = self.emit(cute_ir.make_tile(self.ctx, &.{}, self.tileType(modes), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// `!cute.tile` whose modes are `layouts`, such as `[512:1;1:0;4:1]`. + pub fn tileType(self: *Builder, layouts: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TileType.get(self.ctx, .{ .attr = self.tileAttr(layouts).attribute() }) catch @panic("invalid CuTe tile")).type_(); + } + + fn tileAttr(self: *Builder, layouts: []const LayoutSpec) *const cute_ir.TileAttr { + const modes = self.arena.allocator().alloc(*const cute_ir.TileAttr, layouts.len) catch @panic("OOM"); + for (layouts, modes) |layout, *mode| mode.* = cute_ir.TileAttr.getLayout(self.ctx, self.layoutAttr(layout)) catch @panic("invalid CuTe tile mode"); + return cute_ir.TileAttr.getTuple(self.ctx, modes) catch @panic("invalid CuTe tile"); + } + + /// Construct a target atom whose complete architecture payload is + /// encoded in `atom_type` (for example `!cute_nvgpu.sm100.mma_bs<...>`). + pub fn makeAtom(self: *Builder, atom_type: *const mlir.Type, values: anytype) Atom { + const result = self.emit(cute_ir.make_atom(self.ctx, self.innerSlice(values), atom_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate shared memory with a type assembled by the Zig DSL. + pub fn allocSmemView(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(cute_ir.memref_alloc_smem(self.ctx, memref_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate one compiler-visible shared-memory storage object. The field + /// descriptions use the CuTe host compiler's `name:size:offset` format; + /// they let its shared-memory sizing/layout pass reproduce a Python + /// `SharedStorage` struct while Zig keeps typed pointers to its fields. + pub fn allocSmemStorage( + self: *Builder, + dtype: DType, + alignment: u64, + layout_type: *const mlir.Type, + partition_id: i32, + fields: []const []const u8, + ) View { + const field_attrs = self.arena.allocator().alloc(*const mlir.Attribute, fields.len) catch @panic("OOM"); + for (fields, field_attrs) |field, *attr| attr.* = .string(self.ctx, field); + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(mlir.Operation.make(self.ctx, "cute.memref.alloca", .{ + .results = .{ .flat = &.{memref_type} }, + .attributes = &.{ + .named(self.ctx, "smem.partition_id", .int(self.ctx, .i32, partition_id)), + .named(self.ctx, "smem.struct_fields", .array(self.ctx, field_attrs)), + }, + .location = self.loc(), + })); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate a register-memory tensor whose layout is represented by a + /// normal CuTe layout value. + pub fn allocRmem(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { + const layout_value = self.asValue(layout); + const memref_type = self.memrefType(dtype, .rmem, alignment, layout_value.type_()); + const result = self.emit(cute_ir.memref_alloca(self.ctx, memref_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{ .layout = self.asValue(layout).inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{}, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `projection`, like Python's `proj`, selects the modes of `tile` that + /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { + if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); + const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); + const result = self.emit(cute_ir.local_tile( + self.ctx, + self.asValue(input).inner, + self.asValue(tile).inner, + coord_value, + result_type.?, + .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + // A null result type on the algebra operations below requests dialect inference. + fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { + const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); + return .{ .inner = value.inner, .kernel = self }; + } + + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.local_partition( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + self.makeCoord(index), + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); + const result = self.emit(cute_ir.slice( + self.ctx, + self.asValue(input).inner, + self.makeCoord(coord), + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a tensor with the same iterator and a new layout. This is the + /// tensor form of CuTe's `group_modes`: grouping changes only the layout, + /// while the backing pointer or arithmetic-tuple iterator stays intact. + /// The local CUTLASS schema exposes `cute.group_modes` only for layouts, + /// so expressing the equivalent algebra this way also remains portable. + pub fn withLayout(self: *Builder, input: anytype, layout: anytype) View { + const value = self.asValue(input); + const layout_value = self.asValue(layout); + const input_type = value.type_(); + const result_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() + else + std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); + + const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + memref.ptr_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + desc.desc() + else + unreachable; + const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); + const result = self.emit(cute_ir.group_modes( + self.ctx, + self.asValue(input).inner, + result_type.?, + .int(self.ctx, .i32, begin), + .int(self.ctx, .i32, end), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tiled_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); + const result = self.emit(cute_ir.flat_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.coalesce( + self.ctx, + self.asValue(input).inner, + result_type, + .{ .target_profile = if (target_profile) |target| target.inner else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.filter_zeros( + self.ctx, + self.asValue(input).inner, + result_type.?, + .{ .target_profile = if (target_profile) |target| target.inner else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `cute.get_iter`: the iterator of `input`, of type `result_type`. + pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); + const result = self.emit(cute_ir.add_offset( + self.ctx, + self.asValue(input).inner, + self.asValue(offset).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn recastIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.recast_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.append_to_rank( + self.ctx, + self.asValue(input).inner, + self.asValue(element).inner, + result_type, + .int(self.ctx, .i32, rank), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_copy(self.ctx, atom.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: *const mlir.Type) View { + const op = if (destination) + cute_ir.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()) + else + cute_ir.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()); + const result = self.emit(op); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); + const result = self.emit(cute_ir.tiled_copy_retile( + self.ctx, + tiled.inner, + self.asValue(input).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + const result = self.emit(cute_ir.tiled_mma_partition( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.makeCoord(coord), + result_type.?, + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + const result = self.emit(cute_ir.mma_make_fragment( + self.ctx, + tiled.inner, + self.asValue(input).inner, + result_type.?, + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { + var exec = atom; + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA store atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + if (opts.tma_bar_ptr) |mbar| { + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .gather4, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable gather atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + exec = self.setTmaBarrier(exec, mbar); + } + if (opts.cache_policy) |policy| { + const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) + (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else + @panic("cache_policy requires an executable TMA copy atom"); + const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); + exec = .{ .inner = updated.inner, .kernel = self }; + } + self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); + } + + pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { + self.emitVoid(cute_ir.gemm( + self.ctx, + tiled.inner, + self.asValue(d).inner, + self.innerSlice(a), + self.innerSlice(rhs), + self.asValue(c).inner, + self.loc(), + )); + } + + pub fn vectorExtract(self: *Builder, source: Value, lane: i64, dtype: DType) Value { + return self.emit(vector.extract( + self.ctx, + source.inner, + &.{lane}, + &.{}, + dtype.toMlir(self.ctx), + self.loc(), + )); + } + + // ==================== cute_nvgpu.cpasync: TMA (SM90+) ==================== + // The host function builds tiled TMA atoms (Python `make_tiled_tma_atom`); + // the kernel receives them as grid-constant arguments and makes them + // executable before issuing copies. + + pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; + + fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { + return self.emit(mlir.Operation.make(self.ctx, name, .{ + .operands = .{ .flat = operands }, + .attributes = attributes, + .result_type_inference = true, + .location = self.loc(), + })); + } + + fn valueShape(self: *Builder, value: Value) Value { + if (value.type_().isA(cute_ir.ShapeType) != null) return value; + return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); + } + + fn shapeRank(shape: Value) usize { + return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); + } + + /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout + /// at construction and passes data/index tensors together to partition/copy. + pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { + const Tiler = @TypeOf(cta_tiler); + const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) + self.asValue(cta_tiler) + else if (@typeInfo(Tiler) == .pointer) + self.makeShape(self.dimsExpr(cta_tiler)).value() + else + self.makeShape(cta_tiler).value(); + var smem = self.asValue(smem_layout); + const tiler_rank = shapeRank(self.valueShape(tiler)); + const smem_rank = shapeRank(self.valueShape(smem)); + std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); + if (smem_rank == tiler_rank + 1) { + const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); + for (modes, 0..) |*mode, i| mode.* = @intCast(i); + smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); + } + const shape = self.valueShape(self.asValue(gmem)); + const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); + const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); + var cfg = config; + cfg.mode = .tiled; + if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); + cfg.cta_group = op.cta_group; + if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); + if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); + cfg.mode = .gather4; + const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); + const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); + const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); + const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); + return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; + } + + /// Build the host-visible TMA descriptor and its coordinate tensor, whose + /// types are assembled from `config`. + pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + std.debug.assert(config.mode == .tiled); + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaLoadAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), + .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Host-visible tiled TMA store atom and its coordinate tensor. + pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_store( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaStoreAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + .{ .tma_format = tmaFormatAttribute(self, config) }, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Host-side tiled TMA load atom type described by `config`. + pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ + .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid gather TMA atom")).type_(); + return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ + .kind = tmaLoadKind(config), + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA load atom")).type_(); + } + + /// Executable device-side form of a tiled TMA load atom. `make_exec_tma` + /// preserves the tensor-map payload and adds the launch mode used by + /// `copy`. + pub fn tmaLoadExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = if (config.mode == .gather4) .gather4 else .tiled, + .numCta = ctaCount(config.cta_group), + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA load atom")).type_(); + } + + /// Host-side tiled TMA store atom type described by `config`. + pub fn tmaStoreAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomNonExecTiledTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA store atom")).type_(); + } + + /// Executable device-side form of a tiled TMA store atom. + pub fn tmaStoreExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA store atom")).type_(); + } + + /// The `()` global stride of tiled TMA atoms. + fn emptyStrideType(self: *Builder) *const mlir.Type { + return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); + } + + /// Generic address (i64) of the tensor map in an executable tiled TMA atom. + pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { + const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); + return self.ptrToInt(ptr, .i64); + } + + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { + self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); + } + + pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, .{}, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Attach the mbarrier that an executable TMA load atom completes. + pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { + const result = self.emit(cute_nvgpu.atom_set_value( + self.ctx, + atom.inner, + mbar.inner, + atom.type_(), + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Partition `smem` and `targets` for the TMA atom `descriptor`; the + /// dependent result types are built by the caller. + pub fn tmaPartition( + self: *Builder, + descriptor: Atom, + cta_coord: anytype, + cta_layout: anytype, + smem: anytype, + targets: anytype, + smem_type: *const mlir.Type, + target_types: []const *const mlir.Type, + ) TmaPartition { + if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); + const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); + const op = cute_nvgpu.atom_tma_partition( + self.ctx, + descriptor.inner, + self.makeCoord(cta_coord), + self.asValue(cta_layout).inner, + self.asValue(smem).inner, + self.innerSlice(targets), + smem_type, + target_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; + return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; + } + + // ==================== cute_nvgpu.tcgen05 (SM100) ==================== + + /// Tiled MMA type of the block-scaled atom described by `cfg`. + pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + return (cute_ir.TiledMmaType.get(self.ctx, .{ + .mmaAtom = self.blockScaledAtomType(cfg), + .atomLayoutMNK = self.layoutAttr(self.layoutSpec(.{ 1, 1, 1 }, .{ 0, 0, 0 })).attribute(), + .permutationMNK = self.algebraAttr(cute_ir.TileAttr, self.expr(.{ AlgebraToken.all, AlgebraToken.all, AlgebraToken.all })).attribute(), + }) catch @panic("invalid block-scaled tiled MMA")).type_(); + } + + fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + cfg.validate(); + return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), + .numCta = ctaCount(cfg.cta_group), + .aMajor = .k, + .bMajor = .k, + .aType = DType.f4e2m1fn.toMlir(self.ctx), + .bType = cfg.bType().toMlir(self.ctx), + .cType = DType.f32.toMlir(self.ctx), + .sfType = cfg.scaleType().toMlir(self.ctx), + .aFragKind = .smem_desc, + .vecSize = cfg.vecSize(), + }) catch @panic("invalid block-scaled MMA atom")).type_(); + } + + /// Block-scaled atom and its tiled MMA. Both scale-factor pointers start + /// at `scale_ptr` and are patched after tensor-memory allocation. + pub fn makeBlockScaledMma(self: *Builder, cfg: BlockScaledMmaConfig, scale_ptr: Value) BlockScaledMma { + const disabled = self.constant(.i1, false); + const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); + const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); + return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; + } + + /// Enable (or disable) accumulation into C for the next block-scaled MMA. + pub fn setMmaAccumulate(self: *Builder, mma: Atom, enabled: Value) Atom { + const result = self.emit(cute_nvgpu.atom_set_value( + self.ctx, + mma.inner, + enabled.inner, + mma.type_(), + (cute_nvgpu.AtomMmaFieldSM100BlockScaledAttr.get(self.ctx, .{ .value = .accum_c }) catch @panic("invalid block-scaled MMA field")).attribute(), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const Tcgen05CommitOptions = struct { + /// Must match the group of the operations being tracked. + cta_group: CtaGroup = .one, + /// Arrive at the same barrier offset in each CTA whose rank bit is set. + /// Without a mask, arrive only on the issuing CTA's barrier. + multicast_mask: ?Value = null, + }; + + /// Track this thread's preceding tcgen05 MMA/copy operations of the + /// selected group, signaling one arrival per destination on completion. + /// The same elected thread that issues those operations must commit them. + /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 + /// operations in a kernel must use the same group. + pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { + self.emitVoid(nvvm.tcgen05_commit( + self.ctx, + self.sharedBarrierPtr(mbar).inner, + opts.cta_group.toNvvm(), + if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, + self.loc(), + )); + } + + /// Wait until asynchronous tensor-memory loads have reached registers. + pub fn fenceTmemLoad(self: *Builder) void { + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); + } + + pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; + + pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( + self.ctx, + self.constant(.i32, columns).inner, + self.asValue(holding_ptr).inner, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + self.loc(), + )); + } + + pub fn retrieveTmemPtr(self: *Builder, holding_ptr: anytype, dtype: DType, alignment: u64) Value { + const ptr_type = self.ptrTy(dtype, .tmem, alignment) catch @panic("invalid tensor-memory pointer type"); + return self.emit(cute_nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); + } + + pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); + } + + pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_dealloc_tmem( + self.ctx, + self.asValue(pointer).inner, + self.constant(.i32, columns).inner, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + self.loc(), + )); + } + + pub const TmemLoadOptions = struct { + num_dp: u16, + num_b: u16, + num_rep: u16, + pack_16: bool = false, + }; + + pub fn tmemLoad(self: *Builder, src: anytype, result_type: *const mlir.Type, opts: TmemLoadOptions) View { + const result = self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + result_type, + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Direct tensor-memory load into an SSA vector. Higher-level tiled-copy + /// helpers remain preferable when their dependent layouts are available. + pub fn tmemLoadVector(self: *Builder, src: anytype, lanes: i64, opts: TmemLoadOptions) Value { + // The NVVM-facing operation transports raw 32-bit register words. + // Callers reinterpret each lane as the accumulator element type. + const result_type = mlir.Type.vector(&.{lanes}, DType.i32.toMlir(self.ctx)); + return self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + result_type, + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, + self.loc(), + )); + } + + pub const S2tCopyConfig = struct { + num_dp: i32, + num_bit: i32, + num_cta: i32 = 1, + broadcast: cute_nvgpu.CopyS2TBroadcast = .none, + }; + + /// `!cute_nvgpu.atom.s2t_copy`, a `tcgen05.cp` atom moving `dtype`. + pub fn s2tCopyAtomType(self: *Builder, dtype: DType, cfg: S2tCopyConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomSM100CopyS2TType.get(self.ctx, .{ + .valType = dtype.toMlir(self.ctx), + .numDp = cfg.num_dp, + .numBit = cfg.num_bit, + .numCta = cfg.num_cta, + .broadcast = cfg.broadcast, + }) catch @panic("invalid S2T copy atom")).type_(); + } + + pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_s2t_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_tmem_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_nvgpu.atom_get_copy_s2t_smem_desc_view( + self.ctx, + atom.inner, + self.asValue(input).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Raw SM100 TMA load. The descriptor address, destinations and + /// coordinates come from the executable atom and partition operations. + pub fn tmaLoad( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + barrier_addr: Value, + coords: []const Value, + multicast_mask: ?Value, + offsets: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + num_cta: u8, + ) void { + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_load( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + barrier_addr.inner, + self.innerSlice(coords), + self.innerSlice(offsets), + mode, + .int(self.ctx, .i32, num_cta), + .{ .multicast_mask = if (multicast_mask) |v| v.inner else null, .cache_policy = if (cache_policy) |v| v.inner else null }, + self.loc(), + )); + } + + /// Raw SM100 TMA store. + pub fn tmaStore( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + coords: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + ) void { + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_store( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + self.innerSlice(coords), + mode, + .{ .cache_policy = if (cache_policy) |v| v.inner else null }, + self.loc(), + )); + } + + // ==================== cute.arch ==================== + + fn sreg(self: *Builder, comptime reg: []const u8) Index3 { + var out: [3]Value = undefined; + inline for (.{ "x", "y", "z" }, 0..) |axis, i| { + out[i] = self.emit(nvvm.read_sreg(self.ctx, reg ++ "." ++ axis, self.loc())); + } + return .{ .x = out[0], .y = out[1], .z = out[2] }; + } + + pub fn threadIdx(self: *Builder) Index3 { + return self.sreg("tid"); + } + + pub fn blockIdx(self: *Builder) Index3 { + return self.sreg("ctaid"); + } + + pub fn blockDim(self: *Builder) Index3 { + return self.sreg("ntid"); + } + + pub fn gridDim(self: *Builder) Index3 { + return self.sreg("nctaid"); + } + + /// Mark a scalar as identical in every lane of its warp. CuTe uses this + /// fact when lowering role branches that contain warp-collective TMA and + /// tensor-memory operations. + pub fn makeWarpUniform(self: *Builder, value: anytype) Value { + const scalar = self.asValue(value); + // The CuTe Python frontend legalizes predicates through an i32: + // extui i1 -> i32; make_warp_uniform i32; cmpi ne 0 + // `arch.make_warp_uniform` on i1 parses, but the native SM100 + // pipeline does not lower it all the way to NVVM. + const uniform_input = if (scalar.dtype() == .i1) scalar.to(.i32) else scalar; + const uniform = self.emit(cute_nvgpu.arch_make_warp_uniform( + self.ctx, + uniform_input.inner, + uniform_input.type_(), + self.loc(), + )); + return if (scalar.dtype() == .i1) uniform.ne(0) else uniform; + } + + /// `cute.arch.sync_threads()`. + pub fn syncThreads(self: *Builder) void { + _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); + } + + /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). + /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be + /// a positive warp multiple, and every participating thread must execute + /// the same barrier id. The aligned form requires CTA-uniform control flow. + pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { + if (@TypeOf(participants) == ?Value) { + if (participants) |count| return self.barrierCtaSyncAligned(id, count); + return self.barrierCtaSyncAligned(id, null); + } + if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { + std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); + } + if (@TypeOf(id) == Value) { + std.debug.assert(id.dtype() == .i32); + } else { + const immediate: i32 = id; + std.debug.assert(immediate >= 0 and immediate < 16); + } + // The LLVM intrinsic preserves convergence and both operands across + // the embedded compiler's NVVM dialect version boundary. + if (@TypeOf(participants) == @TypeOf(null)) { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); + } + } + + pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; + + pub fn barrier(self: *Builder, opts: BarrierOptions) void { + self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); + } + + /// Build the CuTe first-class divisor used by CUTLASS's persistent tile + /// scheduler. Its arithmetic is lowered to the same multiply/shift + /// sequence as Python CuTe DSL rather than an integer div/rem pair. + pub fn fastDivmodCreate(self: *Builder, divisor_: anytype) Value { + const divisor = self.lift(divisor_).to(.i32); + return self.emit(cute_ir.fast_divmod_create_divisor( + self.ctx, + divisor.inner, + (cute_ir.FastDivmodDivisorType.get(self.ctx, .{ .width = 32, .isPow2 = false }) catch @panic("invalid fast divmod divisor")).type_(), + self.loc(), + )); + } + + /// Return `{ quotient, remainder }` for a CUTLASS FastDivmod divisor. + pub fn fastDivmod(self: *Builder, dividend_: anytype, divisor: Value) [2]Value { + const dividend = self.lift(dividend_).to(.i32); + return self.emitResults(2, cute_ir.fast_divmod_compute( + self.ctx, + dividend.inner, + divisor.inner, + DType.i32.toMlir(self.ctx), + DType.i32.toMlir(self.ctx), + self.loc(), + )); + } + + /// Elect one lane from the active warp. Used by TMA issue, tensor-memory + /// allocation, and tcgen05 commit operations. + pub fn electSync(self: *Builder) Value { + return self.emit(nvvm.elect_sync(self.ctx, self.loc())); + } + + /// `cute.arch.shuffle_sync_bfly`: exchange a scalar with the lane at + /// `lane_id ^ offset` in the current warp. + /// Warp-wide maximum of a non-negative float, as one `redux.sync` instead + /// of a shuffle reduction: IEEE bit patterns of non-negative floats order + /// like unsigned integers. + pub fn warpMaxNonNegative(self: *Builder, value: Value) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const reduced = self.emit(llvm.inline_asm( + self.ctx, + "redux.sync.max.u32 $0, $1, 0xffffffff;", + &.{bits.inner}, + .{ .constraints = "=r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) reduced else reduced.bitCast(dtype); + } + + pub fn shuffleXor(self: *Builder, value: Value, offset: i32) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.constant(.i32, offset).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// `cute.arch.shuffle_sync`: read a scalar from lane `lane` of the + /// current warp. Every lane must participate. + pub fn shuffleIdx(self: *Builder, value: Value, lane: anytype) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.constant(.i32, lane).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// Four FP32 values rounded to FP8 E4M3 (saturating) and packed into one + /// 32-bit word, `a` in the lowest byte. + pub fn packFp8x4(self: *Builder, a: Value, b: Value, c: Value, d: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }", + &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner }, + .{ .constraints = "=r,f,f,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// Eight FP32 values rounded to FP4 E2M1 (saturating, nearest-even) and + /// packed into one 32-bit word, `values[0]` in the lowest nibble: the + /// packed NVFP4 storage order. + pub fn packFp4x8(self: *Builder, values: [8]Value) Value { + var operands: [8]*const mlir.Value = undefined; + for (&operands, values) |*operand, value| operand.* = value.to(.f32).inner; + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 b0, b1, b2, b3; cvt.rn.satfinite.e2m1x2.f32 b0, $2, $1; cvt.rn.satfinite.e2m1x2.f32 b1, $4, $3; " ++ + "cvt.rn.satfinite.e2m1x2.f32 b2, $6, $5; cvt.rn.satfinite.e2m1x2.f32 b3, $8, $7; mov.b32 $0, {b0, b1, b2, b3}; }", + &operands, + .{ .constraints = "=r,f,f,f,f,f,f,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// Two FP32 values rounded to FP4 E2M1 (saturating, nearest-even) in the + /// low byte of a 32-bit word, `a` in the low nibble. + pub fn packFp4x2(self: *Builder, a: Value, b: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 lo; cvt.rn.satfinite.e2m1x2.f32 lo, $2, $1; cvt.u32.u8 $0, lo; }", + &.{ a.to(.f32).inner, b.to(.f32).inner }, + .{ .constraints = "=r,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// The FP8 E4M3 byte at the low end of `bits` (an i32), decoded to FP32. + pub fn e4m3ToF32(self: *Builder, bits: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hl, hh; .reg .b32 pair; cvt.u16.u32 lo, $1; cvt.rn.f16x2.e4m3x2 pair, lo; mov.b32 {hl, hh}, pair; cvt.f32.f16 $0, hl; }", + &.{bits.inner}, + .{ .constraints = "=f,r", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); + } + + /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. + pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + instruction ++ " $0, $1;", + &.{value.to(.f32).inner}, + .{ .constraints = "=f,f", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); + } + + /// Signal programmatic dependent launches after this grid has consumed + /// its inputs. The caller must also launch the producer/consumer chain + /// with PDL enabled. + pub fn launchDependents(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Wait until the preceding grid has made its programmatic dependency + /// available. This is the consumer-side half of a PDL launch chain. + pub fn waitForDependency(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static + /// shared-memory pointer, sized at compile time. + pub fn allocSmem(self: *Builder, dt: DType, size_in_elems: i64, alignment: ?u64) Value { + const al = alignment orelse @max(1, dtypeBitwidth(dt) / 8); + const ty = self.ptrTy(dt, .smem, al) catch @panic("bad smem pointer"); + return self.emit(cute_nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); + } + + // mbarriers, async-proxy fences and bulk-copy groups. + + /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe + /// shared-memory pointer or a raw `i32` shared-memory address. + fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { + if (mbar.isInt()) return self.addressPtr(mbar, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + } + + /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. + pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); + } + + /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a + /// TMA producer does before issuing its copies. + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); + } + + pub const MBarrierArriveOptions = struct { + peer_cta_rank_in_cluster: ?Value = null, + arrive_count: ?Value = null, + scope: MBarrierScope = .cta, + relaxed: bool = false, + }; + + /// Arrive locally or at the same offset in a peer CTA. Local arrivals + /// return the opaque state token; remote arrivals have no state result. + pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { + var ptr = self.sharedBarrierPtr(mbar); + if (opts.peer_cta_rank_in_cluster) |rank| { + ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ + .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, + .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { + .cta => .cta, + .cluster => .cluster, + } }) catch unreachable).attribute(), + .relaxed = .boolean(self.ctx, opts.relaxed), + .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, + }, self.loc()); + if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); + self.emitVoid(op); + return null; + } + + pub const MBarrierTryWaitOptions = struct { + scope: MBarrierScope = .cta, + time_limit: ?Value = null, + }; + + /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend + /// briefly and return false before completion. `scope` selects which + /// threads this wait can synchronize with; the barrier remains local. + pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { + const ptr = self.sharedBarrierPtr(mbar); + if (opts.time_limit) |ticks| { + return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); + } + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); + } + + pub const MBarrierWaitOptions = struct { + scope: MBarrierScope = .cta, + /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. + time_limit: u32 = 10_000_000, + /// Probe before suspending, as in CuTe's pipeline consumer wait. + test_first: bool = true, + }; + + fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + const ptr = self.sharedBarrierPtr(mbar); + const ticks = self.constant(.i32, opts.time_limit); + switch (opts.scope) { + // Preserve the operation recognized by CuTe's pipeline compiler. + .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), + .cluster => { + var pending = self.openWhile(.{}, .{}); + const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); + pending.yieldBefore(ready.eq(false), .{}); + pending.yieldAfter(.{}); + }, + } + } + + /// Acquire a pipeline phase at `scope`: test without suspending, then + /// block only when the phase is still outstanding. The initial test matters when + /// a producer reaches a full ring and must yield to the consumer that + /// recycles its next stage. + pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + if (opts.test_first) { + const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); + var pending = self.openIf(ready.eq(false)); + self.mbarrierWaitLoop(mbar, phase, opts); + pending.close(); + } else { + self.mbarrierWaitLoop(mbar, phase, opts); + } + } + + /// Make mbarrier initialization visible to the async proxy (SM90+). + pub fn fenceMbarrierInit(self: *Builder) void { + self.emitVoid(nvvm.fence_mbarrier_init(self.ctx, self.loc())); + } + + /// Publish ordinary shared-memory writes to the asynchronous (TMA) proxy (SM90+). + pub fn fenceProxyAsyncShared(self: *Builder) void { + self.emitVoid(nvvm.fence_proxy(self.ctx, .@"async.shared", .cta, self.loc())); + } + + /// Close the current group of bulk asynchronous copies, e.g. TMA stores (SM90+). + pub fn cpAsyncBulkCommitGroup(self: *Builder) void { + self.emitVoid(nvvm.cp_async_bulk_commit_group(self.ctx, self.loc())); + } + + /// Wait until at most `pending` bulk copy groups are in flight; with + /// `read`, only until their sources have been read (SM90+). + pub fn cpAsyncBulkWaitGroup(self: *Builder, pending: u32, read: bool) void { + self.emitVoid(nvvm.cp_async_bulk_wait_group(self.ctx, pending, read, self.loc())); + } + + // ==================== raw-address SM90/SM100 primitives ==================== + // Hand-scheduled kernels address shared memory with `i32` offsets in the + // shared window, global memory with `i64` addresses and tensor memory with + // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / + // LLVM operations for them directly, using inline PTX where needed. + + pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; + + /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. + pub fn addressPtr(self: *Builder, address: Value, comptime space: NVPTXAddressSpace) Value { + return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); + } + + fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { + const word = mlir.Type.int(self.ctx, .i32); + return if (n == 1) word else mlir.Type.vector(&.{n}, word); + } + + fn splitWords(self: *Builder, comptime n: usize, v: Value) [n]Value { + var out: [n]Value = undefined; + if (n == 1) { + out[0] = v; + } else { + for (0..n) |i| out[i] = self.vectorExtract(v, @intCast(i), .i32); + } + return out; + } + + fn joinWords(self: *Builder, comptime n: usize, words: [n]Value) Value { + if (n == 1) return words[0]; + var inner: [n]*const mlir.Value = undefined; + for (words, &inner) |w, *x| x.* = w.inner; + return self.emit(vector.from_elements(self.ctx, &inner, self.wordsType(n), self.loc())); + } + + pub const LoadOptions = struct { + /// Read-only for the whole kernel (global: non-coherent `ld.global.nc`). + invariant: bool = false, + }; + + /// Load `n` (1, 2 or 4) consecutive 32-bit words from a typed pointer. + pub fn loadWordsPtr(self: *Builder, comptime n: usize, ptr: Value, opts: LoadOptions) [n]Value { + const v = self.emit(llvm.load(self.ctx, ptr.inner, self.wordsType(n), .{ .alignment = 4 * n, .invariant = opts.invariant }, self.loc())); + return self.splitWords(n, v); + } + + /// Load `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, opts: LoadOptions) [n]Value { + return self.loadWordsPtr(n, self.addressPtr(address, space), opts); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words through a typed pointer. + pub fn storeWordsPtr(self: *Builder, comptime n: usize, ptr: Value, words: [n]Value) void { + self.emitVoid(llvm.store(self.ctx, self.joinWords(n, words).inner, ptr.inner, .{ .alignment = 4 * n }, self.loc())); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, words: [n]Value) void { + self.storeWordsPtr(n, self.addressPtr(address, space), words); + } + + /// Store the low 16 bits of an `i32`. + pub fn storeHalf(self: *Builder, address: Value, comptime space: NVPTXAddressSpace, word: Value) void { + self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); + } + + /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether + /// `predicate` (i1) is true in any of them. + pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { + return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// `bar.warp.sync 0xffffffff`. + pub fn syncWarp(self: *Builder) void { + self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); + } + + /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). + pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { + std.debug.assert(opts.time_limit == null); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); + } + + /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of + /// the CuTe compiler does not model). + pub fn callIntrinsic(self: *Builder, comptime name: []const u8, operands: []const Value, result: ?*const mlir.Type) ?Value { + const alloc = self.arena.allocator(); + const inner = alloc.alloc(*const mlir.Value, operands.len) catch @panic("OOM"); + for (operands, inner) |o, *x| x.* = o.inner; + const op = llvm.call_intrinsic(self.ctx, name, inner, .{ .result_type = result }, self.loc()); + if (result == null) { + self.emitVoid(op); + return null; + } + return self.emit(op); + } + + /// 16-byte `cp.async.cg` global -> shared. With `valid` (i1) false, the destination is + /// zero-filled and `src` is not read (src-size 0). + pub fn cpAsync16(self: *Builder, dst: Value, src: Value, valid: ?Value) void { + const d = self.addressPtr(dst, .shared); + const g = self.addressPtr(src, .global); + if (valid) |v| { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.constant(.i32, 16), self.constant(.i32, 0)) }, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16", &.{ d, g }, null); + } + } + + /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with + /// `noinc`, the arrival is counted in the barrier's expected count. + pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); + } + + /// Wait until all of this thread's cp.async copies have landed. + pub fn cpAsyncWaitAll(self: *Builder) void { + self.emitVoid(nvvm.cp_async_commit_group(self.ctx, self.loc())); + self.emitVoid(nvvm.cp_async_wait_group(self.ctx, 0, self.loc())); + } + + /// Prefetch the global line of `address` (i64) into L2 with evict-last priority. + pub fn prefetchL2EvictLast(self: *Builder, address: Value) void { + _ = self.callIntrinsic("llvm.nvvm.prefetch.global.L2.evict.last", &.{self.addressPtr(address, .global)}, null); + } + + /// Warp-wide TMEM allocation of `columns`; the base address is written to `holder` (i32 smem). + pub fn tmemAllocAt(self: *Builder, holder: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.constant(.i32, columns).inner, self.loc())); + } + + pub fn tmemDeallocAt(self: *Builder, taddr: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.constant(.i32, columns).inner, self.loc())); + } + + pub const Tcgen05Fence = nvvm.Tcgen05FenceKind; + + /// `tcgen05.fence::{before,after}_thread_sync`. + pub fn tcgen05Fence(self: *Builder, comptime kind: Tcgen05Fence) void { + self.emitVoid(nvvm.tcgen05_fence(self.ctx, kind, self.loc())); + } + + /// `tcgen05.wait::st`. + pub fn fenceTmemStore(self: *Builder) void { + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .store, self.loc())); + } + + /// `tcgen05.mma.cta_group::1.kind::f16`: D (TMEM at `d`) = A . B (+ D when + /// `accumulate`, i1), with A and B given by shared-memory descriptors (i64) and the + /// instruction descriptor `idesc` (i32). + pub fn tcgen05MmaF16(self: *Builder, d: Value, adesc: Value, bdesc: Value, idesc: Value, accumulate: Value) void { + self.emitVoid(nvvm.tcgen05_mma(self.ctx, self.addressPtr(d, .tensor).inner, adesc.inner, bdesc.inner, idesc.inner, accumulate.inner, .f16, .cta_1, self.loc())); + } + + /// `tcgen05.ld.32x32b.x{n}`: thread i of the warp gets TMEM lane (warp base + i), + /// `n` consecutive 32-bit columns from `taddr`. + pub fn tmemLoad32x32b(self: *Builder, comptime n: usize, taddr: Value) [n]Value { + const v = self.emit(nvvm.tcgen05_ld(self.ctx, self.addressPtr(taddr, .tensor).inner, self.wordsType(n), .shape_32x32b, self.loc())); + return self.splitWords(n, v); + } + + /// `tcgen05.st.32x32b.x{n}` (see `tmemLoad32x32b`). + pub fn tmemStore32x32b(self: *Builder, comptime n: usize, taddr: Value, words: [n]Value) void { + self.emitVoid(nvvm.tcgen05_st(self.ctx, self.addressPtr(taddr, .tensor).inner, self.joinWords(n, words).inner, .shape_32x32b, self.loc())); + } + + /// Rank of this CTA in its cluster. + pub fn clusterCtaRank(self: *Builder) Value { + return self.emit(nvvm.read_sreg(self.ctx, "cluster.ctarank", self.loc())); + } + + /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a + /// `shared::cluster` pointer. + pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { + return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + + /// Full cluster barrier (arrive.release + wait.acquire, aligned). + pub fn clusterSync(self: *Builder) void { + self.emitVoid(nvvm.cluster_arrive(self.ctx, true, self.loc())); + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + /// Warp-wide maximum of an f32 (`redux.sync.max.f32`, SM100). + pub fn warpMaxF32(self: *Builder, value: Value) Value { + return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.constant(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; + } + + /// Fast approximate 2^x (`ex2.approx.ftz.f32`). + pub fn exp2Approx(self: *Builder, value: Value) Value { + return self.emit(nvvm.ex2(self.ctx, value.inner, true, self.loc())); + } + + /// Fast approximate 1/x (`rcp.approx.ftz.f32`). + pub fn rcpApprox(self: *Builder, value: Value) Value { + return self.emit(nvvm.rcp_approx_ftz(self.ctx, value.inner, self.loc())); + } + + /// `vector<2 x dt>`, e.g. a bf16x2 pair packed in 32 bits. + fn pairType(self: *Builder, comptime dt: DType) *const mlir.Type { + return mlir.Type.vector(&.{2}, dt.toMlir(self.ctx)); + } + + fn bitcastTo(self: *Builder, value: Value, ty: *const mlir.Type) Value { + return self.emit(llvm.bitcast(self.ctx, value.inner, ty, self.loc())); + } + + /// Two FP8 e4m3 (low 16 bits of `pair`, i32) -> bf16x2 packed in an i32. + pub fn e4m3x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e4m3x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Two FP4 e2m1 (the low byte of `pair`, the even element in the low nibble) + /// -> bf16x2 packed in an i32. + pub fn e2m1x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e2m1x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Eight FP4 e2m1 (one word, even elements in the low nibbles) -> four bf16x2 packed + /// in i32s. The pairs are read as the low bytes of the 16-bit halves of `word` and + /// `word >> 8`, which ptxas maps to operand selectors (one shift per word). + pub fn e2m1x8ToBf16x2x4(self: *Builder, word: Value) [4]Value { + const halves = self.pairType(.i16); + const even = self.bitcastTo(word, halves); + const odd = self.bitcastTo(word.shrLogical(8), halves); + const pairs = [4]Value{ + self.vectorExtract(even, 0, .i16), self.vectorExtract(odd, 0, .i16), + self.vectorExtract(even, 1, .i16), self.vectorExtract(odd, 1, .i16), + }; + var out: [4]Value = undefined; + for (pairs, &out) |pair, *o| o.* = self.e2m1x2ToBf16x2(pair); + return out; + } + + /// bf16x2 * bf16x2 (round to nearest), both packed in i32. + pub fn mulBf16x2(self: *Builder, a: Value, b: Value) Value { + const ty = self.pairType(.bf16); + const p = self.emit(llvm.fmul(self.ctx, self.bitcastTo(a, ty).inner, self.bitcastTo(b, ty).inner, ty, self.loc())); + return self.bitcastTo(p, mlir.Type.int(self.ctx, .i32)); + } + + /// Two f32 -> a packed pair of `dt` (bf16 or f16) in an i32, `lo` in the low half. + pub fn packHalf2(self: *Builder, lo: Value, hi: Value, comptime dt: DType) Value { + const pair = self.emit(vector.from_elements(self.ctx, &.{ lo.to(dt).inner, hi.to(dt).inner }, self.pairType(dt), self.loc())); + return self.bitcastTo(pair, mlir.Type.int(self.ctx, .i32)); + } + + /// A packed pair of `dt` (bf16 or f16) in an i32 -> two f32 (low half first). + pub fn unpackHalf2(self: *Builder, word: Value, comptime dt: DType) [2]Value { + const pair = self.bitcastTo(word, self.pairType(dt)); + return .{ self.vectorExtract(pair, 0, dt).to(.f32), self.vectorExtract(pair, 1, dt).to(.f32) }; + } + + /// f32 -> bf16 bits (round to nearest even) in the low 16 bits of an i32. + pub fn f32ToBf16Bits(self: *Builder, value: Value) Value { + return self.emit(arith.extui(self.ctx, value.to(.bf16).bitCast(.i16).inner, mlir.Type.int(self.ctx, .i32), self.loc())); + } + + /// An i32 equal to `value` at run time that the compiler cannot constant-fold + /// (scaled by `%nctaid.z`, which is 1 for 2-D grids): e.g. a loop bound that must + /// keep a loop rolled (small code, fewer instruction-cache misses). + pub fn opaqueI32(self: *Builder, value: i32) Value { + return self.gridDim().z.mul(value); + } + + /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). + /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), + /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). + /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` + /// are element coordinates; row length and out-of-bounds fill come from the + /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. + pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { + std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); + for (r) |row| std.debug.assert(row.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; + + /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. + pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { + if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); + const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); + self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ + .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, + }, self.loc())); + } + + pub fn prefetchL1(self: *Builder, addr: Value) void { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn prefetchTensorMap(self: *Builder, desc: Value) void { + const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const OperandSource = union(enum) { smem: Value, tmem: Value }; + pub const Tcgen05MmaWsOptions = struct { + collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, + collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, + col_b_zero_mask: ?Value = null, + }; + + /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. + /// The caller elects the issuer and commits/waits before consuming D. + pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { + std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); + const adesc = switch (a) { + .smem => |desc| self.smemDescriptorToInt(desc), + .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), + }; + const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); + self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ + .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, + }, self.loc())); + } + + pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { + return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { + .k => .k, + .mn => .mn, + } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); + } + + pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); + } + + pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { + if (desc.isInt()) return desc.to(.i64); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); + } + + pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .increase); + } + + pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .decrease); + } + + fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { + comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); + const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); + _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. + pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { + std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, switch (scope) { + .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. + fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { + if (!ptr.isPtr()) return ptr; + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } + + /// Map a local shared pointer/address to a typed pointer in peer CTA memory. + pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { + const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); + const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem); + const result_type = (cute_ir.PtrType.get(self.ctx, .{ + .valueType = ty.valueType(), + .addressSpace = .dsmem, + .alignment = ty.alignment(), + .swizzle = ty.swizzle(), + .bitlayout = ty.bitlayout(), + }) catch @panic("invalid DSMEM pointer")).type_(); + return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); + } + return remote; + } + + /// Store one, two, or four i32 words in a peer CTA, completing bytes on its + /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. + pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { + const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); + } + if (mbar.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); + } + const packed_value = if (@TypeOf(value) == Value) value else blk: { + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + break :blk self.joinWords(n, words); + }; + self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); + } + + /// Explicit aligned variants retain the convergence requirement of raw PTX. + pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrierClusterWaitAligned(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + pub const ClusterArriveOptions = struct { aligned: bool = false }; + + pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { + if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn clusterWait(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); + } + + /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. + pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// A null mask selects CuTe's FULL_MASK. + pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; + } + + pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; + + pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { + if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; + return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ + .abs = if (opts.abs) .unit(self.ctx) else null, + .nan = if (opts.nan) .unit(self.ctx) else null, + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }, self.loc())); + } + + fn vec2(self: *Builder, x: [2]Value) Value { + const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); + return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); + } + + fn unvec2(self: *Builder, v: Value) [2]Value { + return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; + } + + pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; + + fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { + return .{ + .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }; + } + + pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub const StoreOptions = struct { + /// Address space for raw integer addresses; typed pointers retain theirs. + space: NVPTXAddressSpace = .global, + level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, + }; + + /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. + pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { + if (@TypeOf(value) != Value) { + const n = common.arity(@TypeOf(value), "store: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; + if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + var args: [9]*const mlir.Value = undefined; + args[0] = address.inner; + for (words, 0..) |word, i| args[i + 1] = word.inner; + _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + return; + } + self.store(addr, self.joinWords(n, words), opts); + return; + } + const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { + const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); + break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } else addr; + self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ + .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), + }, self.loc())); + } + + pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { + return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; + } + + pub fn clock(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); + } + + pub fn globalTimer(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); + } + + // ==================== scalars ==================== + + /// `value` as a constant of `dt`; a Value passes through and enums use + /// their backing integers. An integer is signed or an unsigned bit + /// pattern of `dt`'s width; anything else panics. + pub fn constant(self: *Builder, dt: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); + const ctx = self.ctx; + const T = @TypeOf(value); + if (isFloatDtype(dt)) { + const f: f64 = switch (@typeInfo(T)) { + .comptime_int, .comptime_float => value, + .int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), + .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), + }; + return self.emit(switch (dt) { + inline .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), + else => unreachable, + }); + } + const i: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), + }; + return self.emit(arith.constant_int(ctx, intBits(i, @intCast(dtypeBitwidth(dt)), @tagName(dt)), dt.toMlir(ctx), self.loc())); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + fn fitsInt(i: i64, dt: DType) bool { + if (dt == .i1) return i == 0 or i == 1; + const bits = dtypeBitwidth(dt); + if (bits >= 64) return true; + const shift: u6 = @intCast(bits - 1); + return i >= -(@as(i64, 1) << shift) and i < (@as(i64, 1) << shift); + } + + fn literalFits(value: anytype, dt: DType) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int => fitsInt(@intCast(value), dt), + .bool => !isFloatDtype(dt), + else => isFloatDtype(dt), + }; + } + + /// A Zig scalar as a constant: ints become `i32` when they fit, `i64` + /// otherwise; floats `f32`, like the Python DSL's `Int32`/`Float32`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .int => if (literalFits(value, .i32)) self.constant(.i32, value) else self.constant(.i64, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + return self.constant(like.dtype(), value); + } + + fn liftLike(self: *Builder, value: anytype, ref: Value) Value { + const dt = ref.dtype(); + return if (literalFits(value, dt)) self.constant(dt, value) else self.lift(value); + } + + /// Lift a literal to the other operand's dtype when it fits, else to its + /// own width; then widen the narrower integer. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + var av: Value = if (@TypeOf(a) == Value) a else if (@TypeOf(b) == Value) self.liftLike(a, b) else self.lift(a); + var bv: Value = if (@TypeOf(b) == Value) b else self.liftLike(b, av); + if (av.isInt() and bv.isInt()) { + const aw = dtypeBitwidth(av.dtype()); + const bw = dtypeBitwidth(bv.dtype()); + if (aw < bw) av = av.to(bv.dtype()); + if (bw < aw) bv = bv.to(av.dtype()); + } + return .{ av, bv }; + } + + fn cmp(self: *Builder, a: Value, b: anytype, ip: arith.CmpIPredicate, fp: arith.CmpFPredicate) Value { + const l, const r = self.coerce(a, b); + return self.emit(if (l.isFloat()) arith.cmpf(self.ctx, fp, l.inner, r.inner, self.loc()) else arith.cmpi(self.ctx, ip, l.inner, r.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: anytype, f: anytype) Value { + const tv, const fv = self.coerce(t, f); + return self.emit(arith.select(self.ctx, cond.inner, tv.inner, fv.inner, self.loc())); + } + + // ==================== scf ==================== + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + /// `for iv in range(lower, upper, step)` with loop-carried `inits`. Literal + /// bounds take the type of a Value bound, else `lower` is lifted (`i32` + /// when it fits). + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); + } + + /// `scf.while` with explicit loop-carried values. The before region sees + /// `inits`, calls `yieldBefore(condition, forwarded)`, then the after + /// region sees `forwarded` and calls `yieldAfter(next_inits)`. This is the + /// control-flow shape emitted by the Python CuTe persistent scheduler. + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== module ==================== + + /// The module the CuTe compiler takes: the verified kernel as a public + /// `func.func`, which the compiler turns into the kernel entry. Grid and + /// block travel in the custom call; `block` is also pinned as + /// `nvvm.reqntid`. + pub fn finish(self: *Builder, block: [3]i32) FinishError![:0]const u8 { + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(current); + } + const func_op = self.func_op orelse return error.InvalidMlir; + func_op.setAttribute(nvvm.reqntid(self.ctx, block)); + try common.verify(self.module, "dsl/cute"); + + return self.renderModule(); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "naive elementwise add matches the Python notebook kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "naive_elementwise_add_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gB = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gC = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + const n = a.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + a.gC.set(.{ mi, ni }, a.gA.get(.{ mi, ni }).add(a.gB.get(.{ mi, ni }))); + + // The module round-trips through the local dialects. + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"func.func\"() <{function_type = (!cute.ptr>, !cute.ptr>, !cute.ptr>) -> (), sym_name = \"naive_elementwise_add_kernel\"}>"); + try expectContains(ir, "}) {cute.kernel, gpu.kernel, nvvm.reqntid = array} : () -> ()"); + try expectContains(ir, "\"nvvm.read.ptx.sreg.tid.x\"() : () -> i32"); + try expectContains(ir, "!cute.layout<\"(16,8):(8,1)\">"); + try expectContains(ir, "!cute.coord<\"(?,?)\">"); + try expectContains(ir, "cute.memref.load"); + try expectContains(ir, "cute.memref.store"); + try expectContains(ir, "arith.addf"); + try expectContains(ir, "\"func.return\"() : () -> ()\n })"); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "guard, shared memory, sync, layouts, for loop" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "reverse_blocks"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{4096} } }, + }); + const gx = b.makeTensor(a.x, b.makeLayout(&.{4096}, .{})); + const smem = b.makeTensor(b.allocSmem(.f32, 128, 16), b.makeLayout(&.{128}, .{})); + + const tid = b.threadIdx().x; + const i = b.blockIdx().x.mul(128).add(tid); + smem.set(.{tid}, gx.get(.{i})); + b.syncThreads(); + b.barrierCtaSyncAligned(2, 128); + b.launchDependents(); + b.waitForDependency(); + const j = b.constant(.i32, 127).sub(tid); + var guard = b.openIf(i.lt(4096)); + { + var acc = b.openFor(0, 4, 1, .{b.constant(.f32, 0.0)}); + { + acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); + } + a.out.set(.{i}, acc.results[0]); + guard.close(); + } + + const kernel = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); + try expectContains(kernel, "!cute.ptr>"); + try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); + try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(kernel, "value = 2 : i32"); + try expectContains(kernel, "value = 128 : i32"); + try expectContains(kernel, "griddepcontrol.launch_dependents;"); + try expectContains(kernel, "griddepcontrol.wait;"); + try expectContains(kernel, "scf.if"); + try expectContains(kernel, "scf.for"); + try expectContains(kernel, "!cute.layout<\"4096:1\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "left-most default stride and static coordinate leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + const t = b.makeTensor(a.p, b.makeLayout(&.{ 4, 4 }, .{})); + try std.testing.expectEqualSlices(i64, &.{ 1, 4 }, t.layout.stride.constSlice()); + try std.testing.expectEqual(@as(i64, 16), t.size()); + const v = t.get(.{ b.threadIdx().x, 3 }); + try std.testing.expect(v.isFloat()); + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "!cute.layout<\"(4,4):(1,4)\">"); + try expectContains(kernel, "!cute.coord<\"(?,3)\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "casts, select, if-else, integer widening" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .tensor = .{ .dtype = .f16, .shape = &.{8} } } }); + const tid = b.threadIdx().x; + + // A literal that does not fit i32 widens the i32 Value instead of truncating. + const big: u32 = 3_000_000_000; + const wide = tid.add(big); + try std.testing.expectEqual(DType.i64, wide.dtype()); + const bound = tid.lt(@as(i64, 5_000_000_000)); + try std.testing.expectEqual(DType.i1, bound.dtype()); + + const f = tid.to(.f32); + const shuffled = b.shuffleXor(f, 16); + try std.testing.expectEqual(DType.f16, f.to(.f16).dtype()); + try std.testing.expectEqual(DType.f64, f.to(.f64).dtype()); + try std.testing.expectEqual(DType.bf16, f.to(.f16).to(.bf16).dtype()); + try std.testing.expectEqual(DType.i32, f.to(.i32).dtype()); + try std.testing.expectEqual(DType.i8, tid.to(.i8).dtype()); + try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); + + const picked = b.select(bound, shuffled, 2.0); + var scope = b.openIfElse(tid.lt(4), .{.f16}); + { + scope.yieldThen(.{picked.to(.f16)}); + } + { + scope.yieldElse(.{b.constant(.f16, 1)}); + } + a.p.set(.{tid}, scope.results[0]); + + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "arith.extsi"); + try expectContains(kernel, "\"arith.constant\"() <{value = 3000000000 : i64}> : () -> i64"); + try expectContains(kernel, "arith.sitofp"); + try expectContains(kernel, "arith.truncf"); + try expectContains(kernel, "arith.extf"); + try expectContains(kernel, "arith.convertf"); + try expectContains(kernel, "arith.fptosi"); + try expectContains(kernel, "arith.trunci"); + try expectContains(kernel, "arith.extui"); + try expectContains(kernel, "arith.select"); + try expectContains(kernel, "shfl.sync.bfly.b32"); + try expectContains(kernel, "\"scf.if\""); + try expectContains(kernel, "}, {"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "complete program emits a Zig-built CUDA host launch" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.openProgram(std.testing.allocator, ctx, "program_add"); + defer b.deinit(); + const device = try b.declareArgs(.{ + .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + .output = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + }); + const tid = b.threadIdx().x; + device.output.set(.{tid}, device.input.get(.{tid}).add(1.0)); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction("program_add", .host); + _ = try b.declareArgs(.{ + .input = .{ .ptr = DType.f32 }, + .output = .{ .ptr = DType.f32 }, + }); + const one = b.constant(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, one }, + .block = .{ b.constant(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize("program_add"), + .stream = b.cudaStream(), + .use_pdl = true, + }); + const launched = b.launchEx("program_add", config, .{ b.arg(0), b.arg(1) }); + b.returnHostStatus(b.cudaResultStatus(launched)); + b.endFunction(null); + + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"gpu.module\""); + try expectContains(ir, "cuda.kernel"); + try expectContains(ir, "sym_name = \"program_add\""); + try expectContains(ir, "cuda.launch_ex"); +} + +test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); + defer b.deinit(); + const tma: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 16384, + .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), + .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), + .coordinate_rank = 3, + }; + var pair_tma = tma; + pair_tma.cta_group = .two; + const args = try b.declareArgs(.{ + .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, + }); + _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); + _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); + + const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + for ([_]CtaGroup{ .one, .two }) |group| { + _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); + } + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); + const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); + + const ir = try b.finish(.{ 192, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); + try expectContains(ir, "non_exec_tiled_tma_load"); + try expectContains(ir, "value = 3 : i32"); + try expectContains(ir, "value = 32 : i32"); + try expectContains(ir, "!llvm.ptr<7>"); + try expectContains(ir, "#nvvm.mem_scope"); + try expectContains(ir, "#nvvm.mem_scope"); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "mbarrier waits preserve acquire scope through polling and blocking retries" { + const ctx = try testContext(); + defer ctx.deinit(); + for ([_]MBarrierScope{ .cta, .cluster }) |scope| { + var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + const barrier = b.allocSmem(.i64, 1, 8); + const phase = b.constant(.i32, 0); + _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); + _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); + b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); + b.mbarrierWait(barrier, phase, .{ .scope = scope }); + + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try expectContains(ir, "value = 1234 : i32"); + try expectContains(ir, "\"scf.if\""); + switch (scope) { + .cta => { + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); + }, + .cluster => { + try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + }, + } + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + } +} + +test "SM100 raw primitives preserve operand types and round-trip through MLIR" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 1024, + .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const a = try b.declareArgs(.{ + .gmem = .{ .ptr = DType.i32 }, + .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + }); + const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); + const desc = b.tmaDescriptorAddress(atom); + const dst = b.constant(.i32, 1024); + const bar = b.constant(.i32, 0); + const zero = b.constant(.i32, 0); + const gmem = b.ptrToInt(a.gmem, .i64); + const pred = b.threadIdx().x.eq(0); + b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); + b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); + b.prefetchL1(gmem); + b.prefetchTensorMap(desc); + const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); + const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); + b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); + b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); + b.setMaxRegisterIncrease(128); + b.barrierCtaSyncAligned(0, 512); + b.setMaxRegisterDecrease(128); + b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); + b.mbarrierSpin(bar, zero, .cta); + b.mbarrierSpin(bar, zero, .cluster); + const state = b.mbarrierArrive(bar, .{}).?; + const remote = b.mapDsmemPtr(dst, zero); + _ = remote; + _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); + b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); + b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + const any = b.voteAnySync(pred, null); + const mask = b.voteBallotSync(pred, null); + const x = b.constant(.f32, 1.0); + const y = b.constant(.f32, 2.0); + const maximum = b.fmax(x, y, .{}); + const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); + const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); + const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); + const scalar = b.fma(maximum, fused[0], fused[1]); + b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); + + const ir = try b.finish(.{ 512, 1, 1 }); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", + "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", + "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", + "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", + "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", + "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", + "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", + "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", + "%globaltimer", + }) |op| try expectContains(ir, op); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { + try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); + try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); +} + +test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); + defer b.deinit(); + const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); + const addr = b.constant(.i32, 128); + const fields = [_]u64{ + 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes + 0x4002404000010000, // sw128, pattern at byte 128: base 1 + 0x400e404000010000, // sw128, pattern at byte 896: base 7 + 0x00007fff3fff0000, // unswizzled, largest representable offsets + }; + const descs = [_]Value{ + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), + }; + const out = b.ptrToInt(args.out, .i64); + for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + for (fields) |bits| { + const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); + defer std.testing.allocator.free(expected); + try expectContains(ir, expected); + } + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const n = b.blockIdx().x.add(1); + const singleton = b.constant(.i32, 1); + const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); + const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); + const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); + _ = compact; + _ = explicit; + _ = static_nested; + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "((4,?),?):((1,4),?)"); + try expectContains(ir, "(?,16):(1,?)"); + try expectContains(ir, "((4,8),16):((1,4),32)"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves 64-bit shape and stride values" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); + defer b.deinit(); + const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); + const wide = args.extent.add(@as(i64, 1) << 32); + const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); + const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; + try std.testing.expect(explicit.type_().eql(b.layoutType(.{ + .shape = b.expr(.{ 8, 2 }), + .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, + }))); + _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); + try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); + try expectContains(ir, "value = 4294967296 : i64"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const args = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + }); + const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); + const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); + const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); + b.copy(store.atom, .{smem}, .{tile}, .{}); + b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); + const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); + try expectContains(ir, "(16,4):(1@0,1@1)"); + try expectContains(ir, "tma_store"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); + const tid = b.threadIdx().x; + const mbar = b.allocSmem(.i64, 1, 8); + b.barrier(.{}); + b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); + b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); + _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); + const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; + b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); + b.clusterArriveRelaxed(.{}); + b.clusterWait(); + b.clusterArriveRelaxed(.{ .aligned = true }); + b.barrierClusterWaitAligned(); + _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); + _ = b.voteBallotSync(tid.eq(0), 0xff); + const x = tid.to(.f32); + const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); + const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); + b.store(a.out, state.to(.i32), .{}); + b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); + const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); + const raw_desc = b.smemDescriptorToInt(desc); + _ = b.intToSmemDescriptor(raw_desc); + const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); + const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; + try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); + try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); + b.store(mapped, tid, .{}); + const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; + for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); + const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); + b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); + b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); + try expectContains(ir, "nvvm.store.ext"); + try expectContains(ir, "vector<2xi32>"); + try expectContains(ir, "#nvvm.fp_rnd_mode"); + try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + .mode = .gather4, + }; + var pair = cfg; + pair.cta_group = .two; + const args = try b.declareArgs(.{ + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, + }); + const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); + const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); + const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; + const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); + const mbar = b.allocSmem(.i64, 1, 8); + b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ + .tma_bar_ptr = mbar, + .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), + }); + b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); + b.endFunction(.{ 32, 1, 1 }); + + b.beginFunction("gather4_copy", .host); + const host = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + }); + const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const tiler = b.makeTile(&.{ 16, 4 }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); + b.returnHostStatus(b.constant(.i32, 0)); + b.endFunction(null); + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); + try expectContains(ir, "atom.tma_partition"); + try expectContains(ir, "mode = gather4, num_cta = 1"); + try expectContains(ir, "mode = gather4, num_cta = 2"); + try expectContains(ir, "non_exec_2d_gather4_tma_load", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); + const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); + const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); + for (out, 0..) |*o, i| { + o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{packed_.inner} }, + .results = .{ .flat = &.{elem} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, + .location = b.loc(), + })); + } + return out; +} + +pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { + var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); + for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); + return w.written(); +} + +pub fn ldAcq(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); +} + +pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); +} + +pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { + return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); +} + +pub fn fenceAcqRelCta(b: *Builder) void { + asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); +} + +pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return b.constant(dt, value); +} + +/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand +/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. +pub fn zext64(b: *Builder, v: Value) Value { + if (v.dtype() == .i64) return v; + return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); +} + +pub fn i16Of(b: *Builder, v: Value) Value { + return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); +} + +pub fn cst32(b: *Builder, v: anytype) Value { + return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); +} + +pub fn cst64(b: *Builder, v: u64) Value { + return b.constant(.i64, @as(i64, @bitCast(v))); +} + +pub fn getSmIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); +} + +pub fn getLaneIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); +} + +/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. +pub fn getGridIdx(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); +} + +pub fn blockRankInCluster(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); +} + +/// `clock64()` (`mov.u64 %rd, %clock64;`). +pub fn clock64(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); +} + +/// Keep the original election sequence until typed election matches its small-batch performance. +pub fn electOneSync(b: *Builder) Value { + const pred = asmValue(b, + \\{ + \\.reg .b32 %rx; + \\.reg .pred %px; + \\ mov.u32 $0, 0; + \\ elect.sync %rx|%px, $1; + \\@%px mov.s32 $0, 1; + \\} + , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); + return pred.ne(0); +} + +/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). +pub fn trap(b: *Builder) void { + asmVoid(b, "trap;", &.{}, "~{memory}"); +} + +/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may +/// reach it at different program points). +pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { + asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); +} + +/// `cute::cluster_arrive`. +pub fn clusterArrive(b: *Builder) void { + asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); +} + +/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). +pub fn gridDependencySynchronize(b: *Builder) void { + asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::warpgroup_reg_alloc`. +pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `cutlass::arch::warpgroup_reg_dealloc`. +pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `ptx::ld_volatile(const uint64_t*)`. +pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); +} + +/// `ptx::ld_volatile(const uint32_t*)`. +pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const uint32_t*)`. +pub fn ldSharedU32(b: *Builder, smem: Value) Value { + return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const float2*)`. +pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { + return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); +} + +/// `ptx::ld_shared(const float4*)`. +pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); +} + +/// `ptx::ld_shared(const uint4*)`. +pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); +} + +/// `ptx::st_shared(const void*, x, y, z, w)`. +pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { + asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); +} + +/// `ptx::st_shared(const uint32_t*, uint32_t)`. +pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { + asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); +} + +/// `ptx::st_shared(const float2*, float2)`. +pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { + asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); +} + +/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. +pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { + std.debug.assert(num_bytes % 8 == 0); + asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); +} + +/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. +pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { + return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); +} + +/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte +/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its +/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). +pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { + std.debug.assert(words.len % 4 == 0); + const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); + const mapped_dst = mapaShared(b, dst, dst_cta_idx); + var i: usize = 0; + while (i < words.len) : (i += 4) { + asmVoid( + b, + "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", + &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, + "r,r,r,r,r,r,~{memory}", + ); + } +} + +/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. +pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); +} + +/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. +pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); +} + +/// `ClusterBarrier::arrive()` on this CTA's barrier. +pub fn mbarrierArrive(b: *Builder, bar: Value) void { + asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); +} + +/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of +/// cluster CTA `cta_id`. +pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { + asmVoid( + b, + "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", + &.{ bar, cta_id }, + "r,r,~{memory}", + ); +} + +/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA +/// `cta_id`'s barrier, only when `pred == 1`. +pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { + asmVoid( + b, + "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", + &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, + "r,r,r,r,~{memory}", + ); +} + +/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). +pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { + asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); +} + +/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of +/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. +pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, + "r,l,r,r,r,l,~{memory}", + ); +} + +/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. +pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, + "r,l,r,r,r,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of +/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. +pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, + "r,l,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. +pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, + "l,r,r,l,~{memory}", + ); +} + +/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. +pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { + asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. +pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { + asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. +pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { + asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `ptx::tcgen05_before_thread_sync`. +pub fn tcgen05BeforeThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); +} + +/// `ptx::tcgen05_after_thread_sync`. +pub fn tcgen05AfterThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). +pub fn fenceViewAsyncTmemLoad(b: *Builder) void { + asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); +} + +/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so +/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). +pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { + comptime var constraints: []const u8 = ""; + inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; + inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); + constraints = constraints ++ ",~{memory}"; + return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); +} + +/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks +/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value +/// tied through it. +pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { + const n = values.len; + const a = b.arena.allocator(); + var w: std.Io.Writer.Allocating = .init(a); + for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); + for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); + w.writer.writeAll(",~{memory}") catch @panic("OOM"); + return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); +} + +/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets +/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. +pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { + return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); +} + +/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. +pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { + asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); +} + +/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, +/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) +/// nonzero accumulates. +pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { + const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); + asmVoid( + b, + "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", + &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, + "r,l,l,r,r,r,r", + ); +} + +/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, +/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. +pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { + var elected = b.openIf(electOneSync(b)); + { + asmVoid( + b, + "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", + &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, + "r,h,~{memory}", + ); + elected.close(); + } +} + +pub fn bits32(v: Value) Value { + return if (v.dtype() == .i32) v else v.bitCast(.i32); +} + +pub fn fromBits32(v: Value, dt: DType) Value { + return if (dt == .i32) v else v.bitCast(dt); +} + +/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). +pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { + const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. +pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { + const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_up_sync(0xffffffff, v, delta)`. +pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { + const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__reduce_add_sync(0xffffffff, v)` (u32). +pub fn reduceAddSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__reduce_min_sync(0xffffffff, v)` (u32). +pub fn reduceMinSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). +pub fn allSync(b: *Builder, pred: Value) Value { + const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); + return r.ne(0); +} + +pub fn popc(b: *Builder, v: Value) Value { + return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; +} + +/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). +pub fn ffs(b: *Builder, v: Value) Value { + return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); +} + +/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. +pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { + return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); +} + +/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). +pub fn fminf(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +/// `fabsf(x)` (`abs.f32`). +pub fn fabsf(b: *Builder, x: Value) Value { + return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); +} + +/// `math::fast_rcp(x)`. +pub fn fastRcp(b: *Builder, x: Value) Value { + return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); +} + +/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). +pub fn expfFast(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). +pub fn expfFastNeg(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. +pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { + return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); +} + +/// `__hmin2(a, b)` on bf16x2 words. +pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__hmax2(a, b)` on bf16x2 words. +pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). +pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { + return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); +} + +/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with +/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). +pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { + const lo = i16Of(b, bf16x2); + const hi = i16Of(b, bf16x2.shrLogical(16)); + return .{ + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), + }; +} +/// IEEE operations used when staging quantized inputs; no FTZ or contraction. +pub fn divRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} +pub fn mulRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +test "canonical primitives preserve aligned barriers and runtime transaction counts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const bar = b.constant(.i32, 0); + const phase = b.constant(.i32, 0); + const bytes = b.threadIdx().x.add(16); + b.barrierCtaSyncAligned(1, 32); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + b.mbarrierInit(bar, 1); + b.fenceMbarrierInit(); + b.mbarrierArriveExpectTx(bar, bytes); + b.mbarrierArriveExpectTx(bar, 16); + b.mbarrierWait(bar, phase, .{ .test_first = false }); + _ = phase.bitXor(1); + b.fenceProxyAsyncShared(); + b.fenceProxyAsyncShared(); + b.cpAsyncBulkCommitGroup(); + b.cpAsyncBulkWaitGroup(0, false); + _ = b.voteBallotSync(bytes.ne(0), null); + _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); + try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/dsl/cute/dtype.zig b/dsl/cute/dtype.zig new file mode 100644 index 00000000..ae98cb08 --- /dev/null +++ b/dsl/cute/dtype.zig @@ -0,0 +1,67 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Element types a `!cute.ptr` can point at. +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f4e2m1fn, + f8e4m3fn, + f8e5m2, + f8e8m0fnu, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f4e2m1fn => .float(ctx, .f4e2m1fn), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { + inline for (@typeInfo(DType).@"enum".field_names) |f| { + const dt = @field(DType, f); + if (t.eql(dt.toMlir(ctx))) return dt; + } + std.debug.panic("type {f} is not a CuTe DType", .{t}); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/fly/BUILD.bazel b/dsl/fly/BUILD.bazel new file mode 100644 index 00000000..c8bdf854 --- /dev/null +++ b/dsl/fly/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + "layout.zig", + ], + import_name = "dsl/fly/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/fly/PORTING.md b/dsl/fly/PORTING.md new file mode 100644 index 00000000..f7cb19f3 --- /dev/null +++ b/dsl/fly/PORTING.md @@ -0,0 +1,104 @@ +# Porting a FlyDSL Python kernel to Zig + +The Zig DSL builds Fly IR directly. Work from the revision and patches in +[the FlyDSL repository rule](../../third_party/flydsl/repo.bzl), reading +Python sources with `git -C show :`. Check the +[ROCm plugin pin](../../platforms/rocm/rocm.bzl) as well: the emitter and +runtime must agree on the dialect version. FlyDSL HEAD may differ from both. + +## Cheat sheet + +| Python (`flydsl.expr`) | Zig | +|---|---| +| `@flyc.kernel def k(A: Tensor, ...)` | `fly.Kernel(Cfg, .{ .inputs = &.{"a"}, .outputs = &.{"c"}, .run = run })`; `const a = K.args(b)` | +| kernel arg `tiled_copy: fx.TiledCopy` | build it inside `run` (static values cannot be arguments) | +| `fx.thread_idx.x`, `fx.block_idx.x` | `b.threadId(.x)`, `b.blockId(.x)` (`i32`) | +| `A.shape.unpack()` | `a.a.shapeStatic().at(i).leaf.s` | +| `fx.make_layout((8,16),(16,1))` | `fly.L(.{ 8, 16 }, .{ 16, 1 })` (comptime) or `b.static(layout)` | +| `fx.make_ordered_layout((8,16),(1,0))` | `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | +| `fx.make_tile(64, 8)` / tuple divisor `(64, 8)` | `fly.tile(.{ 64, 8 })` / `.{ 64, 8 }` | +| `fx.make_int_tuple((m, tid))`, coords | `b.intTuple(.{ m, tid })` (`Value`s become `?` leaves) | +| `t[None, None, bx, by]` | `t.slice(.{ null, null, bx, by })` | +| `t[(0, None), None, None]` | `t.slice(.{ .{ 0, null }, null, null })` | +| `t[i]` (no `None`) / `t[i] = v` | `t.at(i)` / `t.set(i, v)` | +| `fx.flat_divide(t, tile)` etc. | `t.flatDivide(tile)`, `.logicalDivide`, `.zippedDivide`, `.tiledDivide` | +| `fx.raked_product(a, b)` etc. | `a.rakedProduct(b)`, `.logicalProduct`, ... | +| `fx.right_inverse(l)`, `fx.composition(l, r)` | `l.rightInverse()`, `l.composition(r)` | +| `fx.get_shape(l)`, `fx.size(l)` (emitted) | `l.emitShape()`, `l.emitSize()` | +| `fx.size(t.shape).unpack()` (host int) | `t.sizeStatic()` | +| `fx.select(tup, [0])`, `fx.get(tup, [1])` | `tup.selectModes(&.{0})`, `tup.get(&.{1})` | +| `fx.make_view((0,0), fx.make_identity_layout((M,N)))` | `b.identity(.{ m, n })` | +| `fx.make_view(ptr, layout)` | `ptr.view(layout)` | +| `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | +| `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | +| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | +| `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | +| `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | +| `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | +| `tiled_copy.tile_mn` | `tc.tileMN()` | +| `tiled_mma.tile_size_mnk` etc. | `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma).tileSizeMNK()` | +| `tiled_copy.get_slice(tid).partition_S(t)` | `tc.getSlice(tid).partitionS(t)` | +| `thr_copy.retile(t)` | `thr.retile(t)` | +| `tiled_mma.make_fragment_A(t)` | `tm.makeFragmentA(t)` | +| `fx.make_fragment_like(t[, dtype=fx.Boolean])` | `t.makeFragmentLike(null)` / `t.makeFragmentLike(.i1)` | +| `fx.make_rmem_tensor(8, fx.Float16)` | `b.rmemTensor(fly.L(8, 1), .f16)` | +| `fx.copy(atom, src, dst, pred=p)` | `b.copy(atom, src, dst, .{ .pred = p })` | +| `fx.copy_atom_call(atom, src, dst)` | `b.copyAtomCall(atom, src, dst)` | +| `fx.gemm(atom, d, a, b, c)` | `b.gemm(atom, d, a, bb, c)` | +| `t.load()`, `t.store(v)`, `t.fill(0)` | same names | +| `fx.elem_less(coord, (M, N))` | `coord.elemLess(.{ m, n })` | +| `fx.rocdl.make_buffer_tensor(t)` | `b.bufferTensor(t)` | +| `fx.SharedAllocator().allocate(fx.Array[f32, n, 16])` | `b.sharedArray(.f32, n, 16)` (returns an `n:1` tensor) | +| `gpu.barrier()` | `b.barrier()` | +| `x.shuffle_xor(off, 64)` | `x.shuffleXor(off, 64)` | +| `v.reduce(ReductionOp.ADD)` | `v.reduce(.add)` | +| `v.to(fx.Float32)` | `v.to(.f32)` | +| `x * rrms` (vector × scalar) | `x.mul(rrms)` | +| `fmath.rsqrt(x, fastmath=fast)` | `b.setFastMath(true); b.rsqrt(x)` | +| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.lt(n)`, `cond.select(a, b)` | +| `if tid == 0: ...` | `var s = b.openIf(tid.eq(0)); ...; s.close();` | +| `for i in range_constexpr(n)` | a plain Zig `for (0..n)` (unrolled at emit time) | +| `for i in range(lo, hi)` with carried values | `b.openFor(lo, hi, 1, .{ init })` + `.yield(.{ ... })` | +| `launch(grid=..., block=(256,1,1))` | `K.call(inputs, outputs, .{ .grid = ..., .threads = 256 })` | + +## Method + +1. Read the kernel and its helpers at the pin; note every `fx.*` primitive. +2. Write the Zig `run` top to bottom, preserving operation order and numeric + conversions. For matrix loops, use SSA copy and MMA calls as described + under [control flow and register tensors](README.md#control-flow-and-register-tensors): + `b.copyAtomLoad` returns vectors, `b.mmaAtomCall` returns the accumulator, + and `b.copyAtomStore` writes vectors. Carry accumulator values through + `openFor`; do not put mutable register tensors across control-flow regions. +3. `bazel test //dsl/fly:test` (or a `K.emit` test): the module must + verify and re-parse. A panic from `fly.: cannot create operation` + lists the operand types; compare them with the Python trace. +4. Run on the ROCm box against a host reference + (`bazel test //zml:test --@zml//platforms:rocm=true ...`). + +## Things that differ + +- Python config (`N`, `dtype_str`, `const_expr(...)` branches) becomes the + kernel `Config`; branch on it with plain `if` in `run`. +- Python lists of vectors across an unrolled loop are Zig arrays of `Value`. + `v.insert(i, x)` returns a new vector; `v.extract(i)` reads an element. + `v.bitCast(dtype)` preserves total bits and adjusts vector length. +- Keep unsigned semantics explicit for packed bytes and offsets with + `toUnsigned`, `divUnsigned`, `remUnsigned` and `cmpUnsigned`. +- `v.shuffleVector(rhs, indices)` rearranges vector elements within a thread; + `v.shuffle(.up, offset, width)` and `shuffleXor` exchange values across lanes. +- Runtime `while` loops use `b.openWhile(inits, after_types)`, `yieldBefore` + and `yieldAfter`; use a plain Zig `while` only for loops unrolled at emit time. +- Python `Int32(x)` / `Float32(x)` constants are `b.constant(.i32, x)`; most + arithmetic methods also take a literal directly (`tid.add(256)`). +- Cached scalars from LDS are read with `t.at(i)`; there is no implicit load. +- bf16 outputs use `arith.truncf`; the Python's manual round-to-nearest bit + trick for pre-gfx950 parts is not reproduced. + +## Model kernels + +The [MoE kernels](../../zml/kernels/moe/fly/) include native routing and +MXFP4 GEMV/MFMA implementations. llmd's sparse MLA kernel +(`llmd/kernels/mla/fly/sparse_mla.zig`, downstream) uses SSA +matrix accumulators through attention loops. These are examples of the +builder APIs with model tensor layouts and launch wrappers. diff --git a/dsl/fly/README.md b/dsl/fly/README.md new file mode 100644 index 00000000..687c99dc --- /dev/null +++ b/dsl/fly/README.md @@ -0,0 +1,255 @@ +# ZML FlyDSL DSL — Author's Guide + +A Zig builder for AMD's FlyDSL (`fly` / `fly_rocdl` MLIR dialects) that reads +like `flydsl.expr` Python while staying in pure Zig. Kernels are compiled and +launched by ZML's ROCm PJRT plugin through the `__gpu$xla.gpu.fly` custom +call; ZML never links a GPU compiler. + +Two layers define the surface: + +- **Low layer** — `dsl/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, + `TiledMma`, the comptime layout literals in `layout.zig`, and `DType`. + `Builder.open` takes an `*mlir.Context` from `zml.kernel.fly.newContext()`. +- **High layer** — `zml.kernel.fly.Kernel(Config, spec)`: names the kernel's + inputs and outputs, hands `run` its arguments by name (`K.args(b)`), and + `call`s it from a ZML model. + +The dialect bindings construct typed MLIR operations. Layout-transforming +operations use FlyDSL's `inferReturnTypes`; static layout literals use typed +attribute constructors and accessors. + +## Skeleton of a kernel + +```zig +const zml = @import("zml"); +const fly = zml.kernel.fly; + +pub const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), // 128 threads + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), // f32x4 along N +}; + +pub const VectorAdd = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, +}); + +fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = VectorAdd.args(b); // .a/.b/.c: row-major !fly.memref views + const shape = a.a.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, cfg.thr, cfg.val); // fx.make_tiled_copy_tv + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const tile = tc.tileMN(); + const gA = a.a.flatDivide(tile).slice(.{ null, null, bx, by }); // A[None, None, bx, by] + const gB = a.b.flatDivide(tile).slice(.{ null, null, bx, by }); + const gC = a.c.flatDivide(tile).slice(.{ null, null, bx, by }); + const cC = b.identity(.{ m, n }).flatDivide(tile).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + for (0..@intCast(pC.sizeStatic())) |i| pC.set(i, tcC.at(i).elemLess(.{ m, n })); + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +// In a model: +const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ grid_m, grid_n, 1 }, + .threads = 128, +}).c; +``` + +Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` +and `zml/kernel/fly/test.zig`. + +Model kernels live beside their consumers: + +- [MoE routing and MXFP4 projections](../../zml/kernels/moe/fly/moe.zig). +- Sparse MLA, downstream in llmd (`llmd/kernels/mla/fly/sparse_mla.zig`). + +Fly is a separate backend. The [MoE selector](../../zml/kernels/moe/moe.zig) and +llmd's sparse-MLA selector select by platform, +architecture and dtype. The current native kernels target gfx942; their +callers check shape support before routing or launching kernels and use +Triton for unsupported contracts. + +## The ABI + +`Kernel.emit(allocator, cfg, shapes)` takes shapes in input-then-output order +and produces + +```mlir +module attributes {gpu.container_module} { + gpu.module @zml_fly_kernels { + gpu.func @name(%a: !fly.ptr, ..., %c: !fly.ptr) kernel { ... gpu.return } + } +} +``` + +with one `!fly.ptr` per input, then per output. The plugin checks only the +argument **count**, so the pointer element type is on you: `DType.storageElem` +maps predicates to `i8` because that is how XLA stores them. Static DSL values +(layouts, tiles, atoms) must be built inside the body, never passed as +arguments. `Kernel.call` bridges each non-scalar argument to a row-major +`!fly.memref` view of its `zml.Shape`. Rank-zero arguments remain pointers; +use `ptrLoad` and `ptrStore` for their scalar values. + +Launch geometry lives in `CallOpts`: `grid`, `threads` (threads per block, +converted to wavefronts using the device's wave size — 64 on CDNA, 32 on +RDNA — so it must be a multiple of that), optional `waves_per_eu` (sets the +LLVM `amdgpu-waves-per-eu` attribute to `N, N` when positive), +`shared_mem_bytes` (only for `fly.get_dyn_shared`; static LDS from +`sharedArray` needs none) and `zeroed_args` (indices into inputs-then-outputs; +ignored inside HIP graphs, so zero what you need yourself). + +`output_operand_aliases` maps output names to input names for buffers that +the kernel reads and writes. Aliasing preserves the input-then-output ABI; +it does not initialize output memory. + +## Layouts, tiles, coordinates + +`layout.zig` holds comptime literals. `toAttr` turns one into the matching +fly attribute through the dialect's own constructors, and reading a type back +goes through its accessors — neither direction goes through text: + +| Zig | MLIR | +|---|---| +| `fly.it(.{ .{ 2, 4 }, 8 })` | `!fly.int_tuple<((2,4),8)>` | +| `fly.L(.{ 4, 8 }, .{ 1, 4 })` | `!fly.layout<(4,8):(1,4)>` | +| `fly.rowMajor(.{ 4, 8 })` / `fly.colMajor` | `(4,8):(8,1)` / `(4,8):(1,4)` | +| `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | `fx.make_ordered_layout((8,16),(1,0))`, folded | +| `fly.tile(.{ 128, 64 })`, `fly.tile(.{ null, 8 })` | `!fly.tile<[128\|64]>`, `[*\|8]` | + +`b.static(literal)` materializes one as `fly.static`. Wherever the Python +takes a tuple, the Zig takes a tuple literal: `t.flatDivide(.{ 64, 8 })` is a +tile, `t.slice(.{ null, bid })` is `t[None, bid]`, `b.intTuple(.{ m, tid })` +mixes static ints, `null` (`*`) and dynamic `Value`s (`?` leaves, passed as +operands). `v.shapeStatic()` / `v.sizeStatic()` / `v.elemDType()` read the +inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. + +## Atoms and tiled ops + +- `b.copyAtom(.{ .universal = 128 }, .f32)`, `.{ .buffer_copy = 32 }`, + `.{ .buffer_copy_lds = 128 }`. +- `b.mmaAtom(mma_op)` takes an MLIR type. Generated MMA type constructors + return a typed wrapper; handle their error and pass the wrapper's + `.type_()` result. +- `b.tiledCopyTV(atom, thr, val)`, `b.tiledCopy(atom, layout_tv, tile)`, + `b.tiledCopyA/B/C(copy_atom, tiled_mma)`; `tc.getSlice(tid).partitionS/D`, + `.retile`. +- `b.tiledMma(atom, fly.L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null)`; + `tm.makeFragmentA/B/C(t)`, `tm.getSlice(tid).partitionA/B/C`. +- `b.copy(atom, src, dst, .{ .pred = p })`, `b.copyAtomCall(atom, src, dst)`, + `b.gemm(atom, d, a, b, c)`. +- `b.mmaAtomCall(atom, a, b, c)` takes SSA operands and returns the updated + accumulator. Operand vectors must match the atom's per-lane fragment + types and layout; this call performs no tiled partitioning. +- `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over + a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. +- Derived layouts come off the type: `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma)` + then `.tileSizeMNK()`, `.tiledThrValLayoutA()`, ... + +## Memory and scalars + +`t.load()` / `t.store(vec)` move a whole register tensor as a `vector`; +`t.at(coord)` / `t.set(coord, v)` are element accesses (an int tuple for a +coordinate tensor); `t.fill(0)` splats. `b.rmemTensor(layout, dtype)` is +`fx.make_rmem_tensor`; `b.sharedArray(.f32, n, 16)` is a static LDS array +(`fx.SharedAllocator().allocate(fx.Array[...])`). + +Scalars and vectors: `add/sub/mul/div/rem` and `lt(x)` (`cmp` for integers, +`cmpf` for floats; `cmpUnsigned`) broadcast a scalar against a vector; +`to(dtype)`, `cond.select(a, b)`, `v.reduce(.add)`, `v.shuffleXor(off, 64)`, +`b.rsqrt(x)`, `b.constant(.f32, 1.0)`, `b.threadId(.x)` / `b.blockId(.x)` +(already `i32`), `b.barrier()`. + +Integer types are signless in MLIR. `div`, `rem`, `cmp` and `to` use signed +integer semantics, except that `to` treats predicates as unsigned. +`divUnsigned`, `remUnsigned`, `cmpUnsigned` and `toUnsigned` make unsigned +operations explicit. `bitAnd`, `bitOr`, `bitXor`, `shl` and `shrU` operate +on integer bits. + +`splat` creates a vector, including when its length is one. Numeric casts +preserve vector shape. `bitCast` preserves total bits and adjusts vector +length to the destination element width; scalar bitcasts require equal +widths. Use `extract` to obtain a scalar, `insert` to replace an element and +`shuffleVector` to select elements from two vectors. Lane `shuffle` +supports `xor`, `up`, `down` and `idx` modes with a literal or an `i32` +`Value` offset. + +`maximum` propagates NaNs; `maxNum` selects the numeric operand when only +one operand is NaN. `neg`, `Builder.log` and `Builder.fma` provide floating +operations. Fast math is off by default; `setFastMath(true)` tags subsequent +float operations `fastmath` and permits relaxed numerical semantics. + +`Builder.perm` emits AMD byte permutation. `Builder.cvtPkF32Fp8` decodes +two FP8 bytes from the low or high half of an `i32` word using the CDNA3 +packed conversion instruction. Architecture selection belongs in the +backend before these operations are emitted. + +`Builder.ptrAtomicAdd` adds an `i32` counter and returns its old value. +Use `workgroup` scope for LDS pointers and `agent` scope for global +pointers. The ordering is monotonic; callers provide barriers or kernel +dependencies before consuming completed counters. + +Control flow uses `dsl/common`: `b.openFor(lo, hi, step, .{ inits })` +with `.yield(.{ ... })` (`.close()` without inits), `b.openIf(cond)` with +`.close()`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse` +(`.elseBranch()` / `.close()` without types). Types are dtypes, Values for +their types, or MLIR types; tuples can also be arrays. +`openWhile` has separate condition and body regions: `before_carried` +feeds `yieldBefore`, `after_carried` feeds `yieldAfter`, and `results` +contains the values returned when the condition becomes false. + +## Control flow and register tensors + +The published ROCm plugin lowers `scf` to `cf` **before** running the Fly +passes on a native `gpu.func` entry, and Fly's register-alloca promotion +only rewrites a single-block body. Kernels with runtime control flow must +keep their register state in SSA values: + +- `copyAtomLoad` returns a vector and `copyAtomStore` stores a vector. +- `mmaAtomCall` returns an accumulator without allocating register memory. +- Pass accumulators through `openFor` initial values, `carried`, `yield` + and `results`. Branches and while loops also return SSA values through + their scope objects. + +Put each scope's body in a Zig `{ ... }` block right after `openFor`, +`openIf` or `openWhile`, ending with its `yield` (or `close()` for a scope +without values). `openIfElse` and `openWhile` get one block per region, ending +with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` without results) and +`yieldBefore`/`yieldAfter`. The scope variable stays outside, so its `results` +stay readable after the blocks. + +The MoE and sparse-MLA kernels linked above use this path with the published +runtime. Straight-line kernels can still use `rmemTensor`, +`makeFragmentLike`, `fill` and `gemm`. + +## Debugging + +- A failing `inferReturnTypes` panics with the op name and every operand type + (`fly.make_tiled_copy: cannot create operation ... operand 1: !fly.layout<...>`). +- `Kernel.emit` returns the module text; `bazel test //dsl/fly:test` + builds the example kernels on any host, no GPU needed. +- On the device side, `--xla_dump_to=` dumps the module the plugin saw. diff --git a/dsl/fly/builder.zig b/dsl/fly/builder.zig new file mode 100644 index 00000000..a083d943 --- /dev/null +++ b/dsl/fly/builder.zig @@ -0,0 +1,1978 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const common = @import("dsl/common/builder"); +const fly = @import("mlir/dialects/fly"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const layout = @import("layout.zig"); +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +pub const IntTuple = layout.IntTuple; +pub const Layout = layout.Layout; +pub const Tile = layout.Tile; +pub const it = layout.it; +pub const L = layout.L; +pub const rowMajor = layout.rowMajor; +pub const colMajor = layout.colMajor; +pub const ordered = layout.ordered; +pub const tile = layout.tile; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, scf, math, vector, memref, llvm, gpu, rocdl, ub, fly, fly_rocdl }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +pub const Arch = enum { gfx942, gfx950 }; + +pub const Dim = gpu.Dim; + +pub const gpu_module_name = "zml_fly_kernels"; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +// ============================================================================= +// Value +// ============================================================================= + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Kind = fly.TypeKind; + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("fly.Value has no owning Builder; use Builder.* helpers instead"); + } + + /// Derived, never stored: control_flow.zig builds Values with bare + /// `.{ .inner, .kernel }` literals. + pub fn kind(self: Value) Kind { + return fly.typeKind(self.type_()); + } + + pub fn shapeStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.layoutLikeShape(self.type_())); + } + + /// Host value, not an op; see `emitSize`. + pub fn sizeStatic(self: Value) i64 { + return self.shapeStatic().product() orelse std.debug.panic("fly: `{f}` has a dynamic shape", .{self.type_()}); + } + + pub fn intTupleStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.expect(self.type_(), .int_tuple)); + } + + pub fn elemDType(self: Value) DType { + const t = fly.elemType(self.type_()); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported element type `{f}`", .{t}); + } + + fn isI64(self: Value) bool { + const int_ty = self.type_().isA(mlir.IntegerType) orelse return false; + return int_ty.width() == 64; + } + + fn isFloatElem(self: Value) bool { + return self.scalarElemType().isFloat(); + } + + /// The element type of a vector, or the type itself. + fn scalarElemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return vec.shaped().elementType(); + return self.type_(); + } + + fn scalarDType(self: Value) DType { + const t = self.scalarElemType(); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported scalar type `{f}`", .{t}); + } + + // ------------------------------------------------------- layout algebra + // Every op below takes a Value or a comptime Layout/Tile/tuple literal, + // and FlyDSL infers the result type. + + fn unary(self: Value, comptime build: anytype) Value { + const k = self.kern(); + return k.emit(build(k.ctx, self.inner, k.loc())); + } + + fn binary(self: Value, comptime build: anytype, rhs: anytype) Value { + const k = self.kern(); + const r = k.toStatic(rhs); + return k.emit(build(k.ctx, self.inner, r.inner, k.loc())); + } + + pub fn flatDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.flat_divide, tiler); + } + pub fn logicalDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.logical_divide, tiler); + } + pub fn zippedDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.zipped_divide, tiler); + } + pub fn tiledDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.tiled_divide, tiler); + } + pub fn logicalProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.logical_product, tiler); + } + pub fn zippedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.zipped_product, tiler); + } + pub fn tiledProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.tiled_product, tiler); + } + pub fn flatProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.flat_product, tiler); + } + pub fn blockedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.blocked_product, tiler); + } + pub fn rakedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.raked_product, tiler); + } + pub fn composition(self: Value, inner: anytype) Value { + return self.binary(fly.ops.composition, inner); + } + pub fn rightInverse(self: Value) Value { + return self.unary(fly.ops.right_inverse); + } + pub fn leftInverse(self: Value) Value { + return self.unary(fly.ops.left_inverse); + } + pub fn coalesce(self: Value) Value { + const k = self.kern(); + return k.emit(fly.ops.coalesce(k.ctx, self.inner, .{}, k.loc())); + } + pub fn complement(self: Value) Value { + const k = self.kern(); + return k.emit(fly.ops.complement(k.ctx, self.inner, .{}, k.loc())); + } + + /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal + /// (`.{ null, null, bx, by }`, nesting allowed) or an int-tuple Value. + pub fn slice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.slice(k.ctx, self.inner, c.inner, k.loc())); + } + pub fn dice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.dice(k.ctx, self.inner, c.inner, k.loc())); + } + + /// The sub-tuple / sub-layout at a mode path. + pub fn get(self: Value, mode: []const i32) Value { + const k = self.kern(); + return k.emit(fly.ops.get(k.ctx, self.inner, .denseArray(k.ctx, .i32, mode), k.loc())); + } + pub fn selectModes(self: Value, indices: []const i32) Value { + const k = self.kern(); + return k.emit(fly.ops.select(k.ctx, self.inner, .denseArray(k.ctx, .i32, indices), k.loc())); + } + pub fn take(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.ops.take(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); + } + pub fn group(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.ops.group(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); + } + + /// Emits an op; `shapeStatic` reads the type instead. + pub fn emitShape(self: Value) Value { + return self.unary(fly.ops.get_shape); + } + pub fn emitStride(self: Value) Value { + return self.unary(fly.ops.get_stride); + } + pub fn emitLayout(self: Value) Value { + return self.unary(fly.ops.get_layout); + } + pub fn emitIter(self: Value) Value { + return self.unary(fly.ops.get_iter); + } + pub fn emitSize(self: Value) Value { + return self.unary(fly.ops.size); + } + pub fn cosize(self: Value) Value { + return self.unary(fly.ops.cosize); + } + pub fn productEach(self: Value) Value { + return self.unary(fly.ops.int_tuple_product_each); + } + + /// A register tensor shaped like `self`. + pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { + const k = self.kern(); + const dtype_attr: ?*const mlir.Attribute = if (dtype) |d| .typeAttr(d.toMlir(k.ctx)) else null; + return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, .{ .dtype = dtype_attr }, k.loc())); + } + + pub fn elemLess(self: Value, rhs: anytype) Value { + const k = self.kern(); + const r = k.intTuple(rhs); + return k.emit(fly.ops.elem_less(k.ctx, self.inner, r.inner, .int(k.ctx, .i1), k.loc())); + } + + // ---------------------------------------------------------------- tensor access + + /// `t[coord]` with no `null` (a coord tensor yields an int tuple). + pub fn at(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.memref_load(k.ctx, self.inner, c.inner, k.loc())); + } + + pub fn set(self: Value, coord: anytype, value: Value) void { + const k = self.kern(); + const c = k.intTuple(coord); + k.emitNone(fly.ops.memref_store(k.ctx, value.inner, self.inner, c.inner, k.loc())); + } + + /// The whole register tensor as a `vector`. + pub fn load(self: Value) Value { + return self.unary(fly.ops.memref_load_vec); + } + + pub fn store(self: Value, vector_: Value) void { + const k = self.kern(); + k.emitNone(fly.ops.memref_store_vec(k.ctx, vector_.inner, self.inner, k.loc())); + } + + /// Store a splat vector of the tensor's static size. + pub fn fill(self: Value, value: anytype) void { + const k = self.kern(); + const n = self.sizeStatic(); + const dt = self.elemDType(); + const cst = k.constant(dt, value); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(k.ctx)); + const vec = k.emit(vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); + self.store(vec); + } + + // ---------------------------------------------------------------- pointers + + pub fn addOffset(self: Value, offset: anytype) Value { + const k = self.kern(); + const o = k.intTuple(offset); + return k.emit(fly.ops.add_offset(k.ctx, self.inner, o.inner, k.loc())); + } + + /// Scalar load. + pub fn ptrLoad(self: Value) Value { + const k = self.kern(); + const elem = self.elemDType().toMlir(k.ctx); + return k.emit(fly.ops.ptr_load(k.ctx, self.inner, elem, k.loc())); + } + + pub fn ptrStore(self: Value, value: Value) void { + const k = self.kern(); + k.emitNone(fly.ops.ptr_store(k.ctx, value.inner, self.inner, k.loc())); + } + + /// A tensor over this pointer. + pub fn view(self: Value, lay: anytype) Value { + return self.binary(fly.ops.make_view, lay); + } + + // ---------------------------------------------------------------- arithmetic + + /// Broadcast a scalar against a vector operand. + fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { + const k = self.kern(); + const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constantLike(rhs, self); + if (self.type_().eql(r.type_())) return .{ self, r }; + if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) + return .{ self, k.emit(vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; + if (self.type_().isA(mlir.VectorType) == null and r.type_().isA(mlir.VectorType) != null) + return .{ k.emit(vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; + std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + /// Signed. + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shrU(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn divUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.divui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn remUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn cmpUnsigned(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .ult, + .le => .ule, + .gt => .ugt, + .ge => .uge, + }; + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return k.emitFast(arith.negf(k.ctx, self.inner, k.loc())); + } + /// Propagates NaNs; use `maxNum` to select the numeric operand instead. + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maximumf(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn maxNum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Interpret integer operands and results as unsigned. + pub fn toUnsigned(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + if (src.isFloat() and dtype.isFloat()) return self.to(dtype); + const result = self.withElemType(dtype.toMlir(k.ctx)); + const op = if (src.isFloat()) + arith.fptoui(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + arith.uitofp(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() > src.bitWidth()) + arith.extui(k.ctx, self.inner, result, k.loc()) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub fn splat(self: Value, n: i64) Value { + const k = self.kern(); + std.debug.assert(self.type_().isA(mlir.VectorType) == null and n > 0); + return k.emit(vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); + } + + /// Preserve total bits, adjusting vector length to the destination element. + pub fn bitCast(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (self.type_().isA(mlir.VectorType)) |vec| { + const n = @divExact(vec.dimension(0) * @as(i64, src.bitWidth()), dtype.bitWidth()); + return k.emit(vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); + } + std.debug.assert(src.bitWidth() == dtype.bitWidth()); + return k.emit(arith.bitcast(k.ctx, self.inner, dtype.toMlir(k.ctx), k.loc())); + } + + pub fn insert(self: Value, index: i64, value: Value) Value { + const k = self.kern(); + return k.emit(vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); + } + + pub fn shuffleVector(self: Value, rhs: Value, indices: []const i64) Value { + const k = self.kern(); + return k.emit(vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); + } + + fn withElemType(self: Value, elem: *const mlir.Type) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return .vector(&.{vec.dimension(0)}, elem); + return elem; + } + + pub fn toI32(self: Value) Value { + const k = self.kern(); + return k.emit(arith.index_cast(k.ctx, self.inner, .int(k.ctx, .i32), k.loc())); + } + + /// 1 for a scalar. + pub fn vectorLen(self: Value) i64 { + const vec = self.type_().isA(mlir.VectorType) orelse return 1; + return vec.dimension(0); + } + + /// Numeric conversion, scalar or vector. `i1` is treated as unsigned, so a + /// predicate converts to 1, not -1; equal-width float pairs (f16 <-> bf16, + /// the fp8 spellings) go through `arith.convertf`, which is the only op + /// that accepts them. + pub fn to(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + const result = self.withElemType(dtype.toMlir(k.ctx)); + const unsigned_src = src == .i1; + const op = if (src.isFloat() and dtype.isFloat()) + (if (dtype.bitWidth() > src.bitWidth()) + arith.extf(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() < src.bitWidth()) + arith.truncf(k.ctx, self.inner, result, k.loc()) + else + arith.convertf(k.ctx, self.inner, result, .{}, k.loc())) + else if (src.isFloat()) + arith.fptosi(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + (if (unsigned_src) arith.uitofp(k.ctx, self.inner, result, k.loc()) else arith.sitofp(k.ctx, self.inner, result, k.loc())) + else if (dtype.bitWidth() > src.bitWidth()) + (if (unsigned_src) arith.extui(k.ctx, self.inner, result, k.loc()) else arith.extsi(k.ctx, self.inner, result, k.loc())) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub const Cmp = enum { eq, ne, lt, le, gt, ge }; + + /// Signed integer comparison against a Value or an integer literal; a + /// scalar broadcasts against a vector. + pub fn cmp(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .slt, + .le => .sle, + .gt => .sgt, + .ge => .sge, + }; + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); + } + + /// Ordered. + pub fn cmpf(self: Value, pred: Cmp, rhs: Value) Value { + const k = self.kern(); + const p: arith.CmpFPredicate = switch (pred) { + .eq => .oeq, + .ne => .one, + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + }; + return k.emit(arith.cmpf(k.ctx, p, self.inner, rhs.inner, k.loc())); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(.lt, rhs); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(.le, rhs); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(.gt, rhs); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(.ge, rhs); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(.eq, rhs); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(.ne, rhs); + } + + /// `cmpf` for floats, `cmp` (signed) for integers, `index` included. + fn compare(self: Value, pred: Cmp, rhs: anytype) Value { + if (!self.isFloatElem()) return self.cmp(pred, rhs); + const l, const r = self.coerceArith(rhs); + return l.cmpf(pred, r); + } + + pub fn select(self: Value, a: Value, b: Value) Value { + const k = self.kern(); + return k.emit(arith.select(k.ctx, self.inner, a.inner, b.inner, k.loc())); + } + + /// Lane `k` reads lane `k ^ offset`, within `width` lanes. + pub fn shuffleXor(self: Value, offset: i32, width: i32) Value { + return self.shuffle(.xor, offset, width); + } + + pub const ShuffleMode = gpu.ShuffleMode; + + pub fn shuffle(self: Value, mode: ShuffleMode, offset: anytype, width: i32) Value { + const k = self.kern(); + const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); + const wid = k.constant(.i32, width); + return k.emit(gpu.shuffle(k.ctx, self.inner, off.inner, wid.inner, mode, k.loc())); + } + + pub const ReduceKind = enum { add, mul, minf, maxf, minsi, maxsi, and_, or_, xor }; + + /// Reduce a vector to a scalar. + pub fn reduce(self: Value, reduce_kind: ReduceKind) Value { + const k = self.kern(); + const result = self.scalarElemType(); + const ck: vector.CombiningKind = switch (reduce_kind) { + .add => .add, + .mul => .mul, + .minf => .minnumf, + .maxf => .maxnumf, + .minsi => .minsi, + .maxsi => .maxsi, + .and_ => .@"and", + .or_ => .@"or", + .xor => .xor, + }; + const op = vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); + return k.emitFast(op); + } + + /// Static position. + pub fn extract(self: Value, i: i64) Value { + const k = self.kern(); + return k.emit(vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); + } +}; + +// ============================================================================= +// Tiled copy / MMA handles +// ============================================================================= + +/// A `!fly.tiled_copy` plus the tile it was built with. +pub const TiledCopy = struct { + value: Value, + tile: Value, + + pub fn getSlice(self: TiledCopy, thr_idx: Value) ThrCopy { + return .{ .tiled_copy = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub fn tileMN(self: TiledCopy) Value { + return self.tile; + } +}; + +/// Per-thread slice of a TiledCopy. +pub const ThrCopy = struct { + tiled_copy: Value, + thr: Value, + + pub fn partitionS(self: ThrCopy, src: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_partition_src(k.ctx, self.tiled_copy.inner, src.inner, self.thr.inner, k.loc())); + } + + pub fn partitionD(self: ThrCopy, dst: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_partition_dst(k.ctx, self.tiled_copy.inner, dst.inner, self.thr.inner, k.loc())); + } + + pub fn retile(self: ThrCopy, t: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_retile(k.ctx, self.tiled_copy.inner, t.inner, k.loc())); + } +}; + +pub const TiledMma = struct { + value: Value, + + pub fn getSlice(self: TiledMma, thr_idx: Value) ThrMma { + return .{ .tiled_mma = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub const FragmentOpts = struct { + stages: ?i32 = null, + }; + + fn makeFragment(self: TiledMma, operand: fly.MmaOperand, input: Value, opts: FragmentOpts) Value { + const k = self.value.kern(); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + const stages: ?*const mlir.Attribute = if (opts.stages) |st| .int(k.ctx, .i32, st) else null; + return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, .{ .stages = stages }, k.loc())); + } + + /// Per-thread register fragment for the A operand. + pub fn makeFragmentA(self: TiledMma, a: Value) Value { + return self.makeFragment(.a, a, .{}); + } + pub fn makeFragmentB(self: TiledMma, b: Value) Value { + return self.makeFragment(.b, b, .{}); + } + pub fn makeFragmentC(self: TiledMma, c: Value) Value { + return self.makeFragment(.c, c, .{}); + } + pub fn makeFragmentAOpts(self: TiledMma, a: Value, opts: FragmentOpts) Value { + return self.makeFragment(.a, a, opts); + } + pub fn makeFragmentBOpts(self: TiledMma, b: Value, opts: FragmentOpts) Value { + return self.makeFragment(.b, b, opts); + } +}; + +pub const ThrMma = struct { + tiled_mma: Value, + thr: Value, + + fn partition(self: ThrMma, operand: fly.MmaOperand, t: Value) Value { + const k = self.tiled_mma.kern(); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + return k.emit(fly.ops.tiled_mma_partition(k.ctx, self.tiled_mma.inner, t.inner, self.thr.inner, operand_id, k.loc())); + } + + pub fn partitionA(self: ThrMma, a: Value) Value { + return self.partition(.a, a); + } + pub fn partitionB(self: ThrMma, b: Value) Value { + return self.partition(.b, b); + } + pub fn partitionC(self: ThrMma, c: Value) Value { + return self.partition(.c, c); + } +}; + +// ============================================================================= +// Builder +// ============================================================================= + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + /// `builtin.module attributes {gpu.container_module}`. + module: *mlir.Module, + gpu_module: *mlir.Operation, + gpu_body: *mlir.Block, + name: []const u8, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + arg_specs: []const ArgSpec = &.{}, + /// The `!fly.memref` view for tensor arguments, the raw `!fly.ptr` otherwise. + arg_values: []Value = &.{}, + target: Arch = .gfx942, + fast_math: bool = false, + + pub const ArgSpec = struct { + name: []const u8, + dtype: DType, + space: fly.AddressSpace = .global, + /// Row-major tensor argument; null for a bare pointer. + dims: ?[]const i64 = null, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + const loc_: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc_); + errdefer module.deinit(); + module.operation().setAttribute(gpu.containerModule(ctx)); + + const gpu_body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, gpu_module_name, gpu_body, loc_); + _ = gpu_module.appendTo(module.body()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .gpu_module = gpu_module, + .gpu_body = gpu_body, + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Arguments by name. Each field is `.{ .ptr = dtype }` or + /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".field_names; + var specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const inner = @field(raw, variant); + specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) + .{ .name = f, .dtype = inner } + else if (comptime std.mem.eql(u8, variant, "tensor")) + .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + else + @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + } + try self.declareArgsLow(&specs); + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); + return named; + } + + /// One `!fly.ptr` per spec, in order; tensor + /// arguments are bridged to `!fly.memref` views. + pub fn declareArgsLow(self: *Builder, specs: []const ArgSpec) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + self.arg_specs = try scratch.dupe(ArgSpec, specs); + + const arg_types = try scratch.alloc(*const mlir.Type, specs.len); + const arg_locs = try scratch.alloc(*const mlir.Location, specs.len); + for (arg_types, arg_locs, specs) |*ty_, *a_loc, a| { + ty_.* = self.ptrType(a.dtype.storageElem(), self.addressSpaceAttr(a.space)); + a_loc.* = self.loc().named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const func_op = gpu.func(ctx, .{ + .name = self.name, + .args = arg_types, + .block = entry, + .location = self.loc(), + }); + _ = func_op.appendTo(self.gpu_body); + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + + // The view keeps the storage element type: recasting a predicate + // argument back to `!fly.ptr` would make a copy atom's element + // count `bits / 1` while it still moves `bits / 8` bytes. + self.arg_values = try scratch.alloc(Value, specs.len); + for (specs, 0..) |a, i| { + const raw = self.rawArg(i); + self.arg_values[i] = if (a.dims) |dims| raw.view(self.staticRowMajor(dims)) else raw; + } + } + + pub fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.rawArg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// A tensor view when the spec has dims, else the raw pointer. + pub fn arg(self: *Builder, i: usize) Value { + return self.arg_values[i]; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitNone(self: *Builder, op: *mlir.Operation) void { + _ = op.appendTo(self.currentBlock()); + } + + // ---------------------------------------------------------------- types + + /// Arena-allocated. + pub fn readIntTuple(self: *Builder, tuple_ty: *const fly.types.IntTupleType) IntTuple { + if (tuple_ty.isLeaf()) { + return switch (tuple_ty.leaf()) { + .static => |v| IntTuple.static(v), + .dynamic, .basis => IntTuple.dyn, + .none => IntTuple.star, + }; + } + const n = tuple_ty.numElements(); + const elems = self.alloc(IntTuple, n); + for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.element(i)); + return .{ .tup = elems }; + } + + fn alloc(self: *Builder, comptime T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch @panic("fly: OOM"); + } + + /// The shape and stride of a static `!fly.layout`, as host literals. + pub fn readLayout(self: *Builder, layout_ty: *const mlir.Type) Layout { + const lay = fly.expect(layout_ty, .layout); + const shape = lay.shape(); + const stride = lay.stride(); + if (!shape.isStatic() or !stride.isStatic()) { + std.debug.panic("fly: `{f}` is not static", .{layout_ty}); + } + return .{ .shape = self.readIntTuple(shape), .stride = self.readIntTuple(stride) }; + } + + /// A fly type or attribute is rejected only on malformed input, which is + /// a bug in the kernel, not a runtime condition. + pub fn must(result: anytype) @typeInfo(@TypeOf(result)).error_union.payload { + return result catch |err| std.debug.panic("fly: cannot build type ({})", .{err}); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn intTupleType(self: *Builder, t: IntTuple) *const mlir.Type { + return must(fly.types.IntTupleType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn layoutType(self: *Builder, lay: Layout) *const mlir.Type { + return must(fly.types.LayoutType.get(self.ctx, .{ .attr = must(lay.toAttr(self.ctx)) })).type_(); + } + + pub fn tileType(self: *Builder, t: Tile) *const mlir.Type { + return must(fly.types.TileType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn ptrType(self: *Builder, dtype: DType, space: *const mlir.Attribute) *const mlir.Type { + return must(fly.types.PointerType.get(self.ctx, .{ .elemTy = dtype.toMlir(self.ctx), .addressSpace = space })).type_(); + } + + fn addressSpaceAttr(self: *Builder, space: fly.AddressSpace) *const mlir.Attribute { + return must(fly.attributes.AddressSpaceAttr.get(self.ctx, .{ .addressSpace = space })).attribute(); + } + + /// `!fly.memref`. + pub fn memRefType(self: *Builder, dtype: DType, space: fly.AddressSpace, lay: Layout) *const mlir.Type { + return must(fly.types.MemRefType.get(self.ctx, .{ + .elemTy = dtype.toMlir(self.ctx), + .addressSpace = self.addressSpaceAttr(space), + .layout = must(lay.toAttr(self.ctx)).attribute(), + })).type_(); + } + + // ---------------------------------------------------------------- statics + + /// `fly.static` of an already-built type. + pub fn staticTy(self: *Builder, t: *const mlir.Type) Value { + return self.emit(fly.ops.static(self.ctx, t, self.loc())); + } + + pub fn static(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + return self.staticTy(if (T == Layout) + self.layoutType(v) + else if (T == Tile) + self.tileType(v) + else if (T == IntTuple) + self.intTupleType(v) + else + @compileError("fly.static: expected a Layout, Tile or IntTuple, got " ++ @typeName(T))); + } + + pub fn staticRowMajor(self: *Builder, dims: []const i64) Value { + if (dims.len == 1) return self.static(Layout{ .shape = .static(dims[0]), .stride = .static(1) }); + const shape = self.alloc(IntTuple, dims.len); + const stride = self.alloc(IntTuple, dims.len); + var acc: i64 = 1; + var i: usize = dims.len; + while (i > 0) { + i -= 1; + shape[i] = .static(dims[i]); + stride[i] = .static(acc); + acc *= dims[i]; + } + return self.static(Layout{ .shape = .{ .tup = shape }, .stride = .{ .tup = stride } }); + } + + /// A Value passes through; a layout literal becomes a `fly.static`, and a + /// bare tuple literal (`.{ 64, 8 }`) is a tile. + pub fn toStatic(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + if (T == Value) return v; + if (T == Layout or T == Tile or T == IntTuple) return self.static(v); + if (@typeInfo(T) == .@"struct" and @typeInfo(T).@"struct".is_tuple) return self.static(tile(v)); + @compileError("fly: expected a Value, Layout, Tile, IntTuple or tuple literal, got " ++ @typeName(T)); + } + + /// A (possibly nested) int tuple from integers (static leaves), `null` + /// (`*`), `IntTuple` literals and `Value`s (dynamic leaves, passed as + /// operands). An `!fly.int_tuple` Value passes through. + pub fn intTuple(self: *Builder, elems: anytype) Value { + if (@TypeOf(elems) == Value and elems.kind() == .int_tuple) return elems; + var operands: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + const spec = self.intTupleSpec(&operands, elems); + const t = self.intTupleType(spec); + return self.emit(fly.ops.make_int_tuple(self.ctx, operands.constSlice(), t, self.loc())); + } + + /// `intTuple`, spelled for coordinates. + pub const coord = intTuple; + + /// The literal for `intTuple`, collecting each dynamic leaf as an operand. + fn intTupleSpec(self: *Builder, operands: *stdx.BoundedArray(*const mlir.Value, 32), v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == Value) { + if (v.kind() == .int_tuple) { + std.debug.panic("fly.intTuple: nested !fly.int_tuple values are not supported; pass scalars", .{}); + } + operands.appendAssumeCapacity(v.inner); + return if (v.isI64()) .dynamic(64, 1) else .dyn; + } + if (T == IntTuple) return v; + if (T == @TypeOf(null)) return .star; + switch (@typeInfo(T)) { + .comptime_int, .int => return .static(@intCast(v)), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); + const elems = self.alloc(IntTuple, s.field_names.len); + inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); + return .{ .tup = elems }; + }, + else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), + } + } + + // ---------------------------------------------------------------- scalars & ids + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. Only the representation `dtype` needs is computed: a float + /// constant outside i64 range (inf, floatMax) must not be forced through + /// `@intFromFloat`, which traps. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + const is_float_lit = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => true, + else => false, + }; + return switch (dtype) { + .f16, .bf16, .f32, .f64 => blk: { + const f: f64 = if (is_float_lit) @floatCast(value) else @floatFromInt(value); + break :blk self.emit(switch (dtype) { + .f16 => arith.constant_float(self.ctx, f, .f16, self.loc()), + .bf16 => arith.constant_float(self.ctx, f, .bf16, self.loc()), + .f32 => arith.constant_float(self.ctx, f, .f32, self.loc()), + else => arith.constant_float(self.ctx, f, .f64, self.loc()), + }); + }, + .i1, .i8, .i16, .i32, .i64 => blk: { + const i: i128 = if (is_float_lit) @intFromFloat(value) else value; + break :blk self.emit(arith.constant_int(self.ctx, intBits(i, @intCast(dtype.bitWidth()), @tagName(dtype)), dtype.toMlir(self.ctx), self.loc())); + }, + else => std.debug.panic("fly.constant: no constant for {s}", .{@tagName(dtype)}), + }; + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// A Zig scalar as a constant: a comptime int is `i32`, or `i64` when it + /// doesn't fit; a runtime int is `i64` past 32 bits, else `i32`; a float + /// keeps its width, and a comptime float is `f32`. A Value passes through. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => self.constant(if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .i32 else .i64, value), + .int => |info| self.constant(if (info.bits > 32) .i64 else .i32, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + pub fn constIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + fn idI32(self: *Builder, op: *mlir.Operation) Value { + return self.emit(op).toI32(); + } + + /// Already cast to `i32`: `fly.make_int_tuple` takes only i32/i64. + pub fn threadId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.thread_id(self.ctx, dim, self.loc())); + } + pub fn blockId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_id(self.ctx, dim, self.loc())); + } + pub fn blockDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_dim(self.ctx, dim, self.loc())); + } + pub fn gridDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.grid_dim(self.ctx, dim, self.loc())); + } + + pub fn barrier(self: *Builder) void { + self.emitNone(gpu.barrier(self.ctx, self.loc())); + } + + /// Tag float arithmetic emitted from now on with `fastmath`. + pub fn setFastMath(self: *Builder, on: bool) void { + self.fast_math = on; + } + + pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { + if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); + } + return self.emit(op); + } + + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emitFast(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emitFast(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emitFast(math.exp(self.ctx, x.inner, self.loc())); + } + + pub fn log(self: *Builder, x: Value) Value { + return self.emitFast(math.log(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c: Value) Value { + return self.emitFast(math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); + } + + /// A matrix atom with SSA operands; unlike `gemm`, this needs no register + /// allocas and can carry its accumulator through control flow. + pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { + return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, .{}, self.loc())); + } + + /// AMD byte permutation. ROCDL has no corresponding operation at our pin. + pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "v_perm_b32 $0, $1, $2, $3", + &.{ hi.inner, lo.inner, selector.inner }, + .{ .constraints = "=v,v,v,v", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. + pub fn cvtPkF32Fp8(self: *Builder, bits: Value, high: bool) Value { + return self.emit(rocdl.cvtPkF32Fp8(self.ctx, bits.inner, high, self.loc())); + } + + pub const AtomicScope = enum { workgroup, agent }; + + /// Monotonic i32 addition to a global or shared pointer. Callers provide + /// barriers or kernel dependencies before reading the completed counters. + pub fn ptrAtomicAdd(self: *Builder, ptr: Value, value: Value, scope: AtomicScope) Value { + std.debug.assert(ptr.elemDType() == .i32 and value.type_().eql(DType.i32.toMlir(self.ctx))); + const space: i32 = switch (scope) { + .workgroup => 3, + .agent => 1, + }; + const llvm_ptr = self.emit(fly.ops.to_llvm_ptr(self.ctx, ptr.inner, .int(self.ctx, .i32, space), self.loc())); + return self.emit(llvm.atomicrmw(self.ctx, llvm_ptr.inner, value.inner, .add, .monotonic, .{ .syncscope = @tagName(scope), .alignment = 4 }, self.loc())); + } + + /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own + /// `fly.make_ptr {allocBytes, allocAlign}` leaf, which the plugin lowers + /// to one LLVM global, so `shared_mem_bytes` may stay 0. + pub fn sharedArray(self: *Builder, dtype: DType, n: i64, alignment: i32) Value { + const bytes: i64 = @divExact(@as(i64, dtype.bitWidth()) * n, 8); + const raw_ty = must(fly.types.PointerType.get(self.ctx, .{ + .elemTy = .int(self.ctx, .i8), + .addressSpace = self.addressSpaceAttr(.shared), + .alignment = must(fly.attributes.AlignAttr.get(self.ctx, .{ .alignment = alignment })), + })).type_(); + const dict: *const mlir.Attribute = .dict(self.ctx, &.{ + .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), + .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), + }); + const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, .{ .dict_attrs = dict }, self.loc())); + const typed = self.recast(raw, dtype); + return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); + } + + /// A single atom-sized copy. + pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { + self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, .{}, self.loc())); + } + + /// One atom-sized load straight into a `vector`. + /// + /// Required over an rmem tensor + `copyAtomCall` + `load` in any kernel + /// with control flow: the plugin lowers `scf` to `cf` before the Fly + /// passes, and Fly's register-alloca promotion only handles one block. + pub fn copyAtomLoad(self: *Builder, atom: Value, src: Value) Value { + const n = src.sizeStatic(); + const dt = src.elemDType(); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); + return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, &.{vec_ty}, .{}, self.loc())); + } + + /// One atom-sized store of a `vector`; see `copyAtomLoad`. + pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { + self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, &.{}, .{ .dst = dst.inner }, self.loc())); + } + + // ---------------------------------------------------------------- tensors + + pub fn makeView(self: *Builder, iter: Value, lay: anytype) Value { + _ = self; + return iter.view(lay); + } + + /// A coordinate tensor: value == logical coord. + pub fn identity(self: *Builder, shape: anytype) Value { + const shp = self.intTuple(shape); + const lay = self.emit(fly.ops.make_identity_layout(self.ctx, shp.inner, self.loc())); + const rank_ = shp.intTupleStatic().rank(); + const zero: IntTuple = if (rank_ == 1) .static(0) else blk: { + const zeros = self.alloc(IntTuple, rank_); + for (zeros) |*z| z.* = .static(0); + break :blk .{ .tup = zeros }; + }; + const base_ty = self.intTupleType(zero); + const base = self.emit(fly.ops.make_int_tuple(self.ctx, &.{}, base_ty, self.loc())); + return base.view(lay); + } + + /// A register tensor; see `copyAtomLoad` before using one in a kernel + /// with control flow. + pub fn rmemTensor(self: *Builder, comptime lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); + } + + /// `rmemTensor` with a runtime `Layout`. + pub fn rmemTensorRuntime(self: *Builder, lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); + } + + /// Reinterpret a pointer as another element type, keeping the address + /// space, alignment and swizzle. + pub fn recast(self: *Builder, ptr: Value, dtype: DType) Value { + const t = must(fly.ptrWithElem(self.ctx, ptr.type_(), dtype.toMlir(self.ctx))).type_(); + return self.emit(fly.ops.recast_iter(self.ctx, ptr.inner, t, self.loc())); + } + + // ---------------------------------------------------------------- atoms + + /// The copy-op type an atom wraps, by bit size. + pub const CopyOp = union(enum) { + universal: i32, + buffer_copy: i32, + buffer_copy_lds: i32, + + fn type_(self: CopyOp, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .universal => |b| must(fly.types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy_lds => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), + }; + } + }; + + pub fn copyAtom(self: *Builder, op: CopyOp, dtype: DType) Value { + const bits: i32 = @intCast(dtype.bitWidth()); + const t = must(fly.types.CopyAtomType.get(self.ctx, .{ .copyOp = op.type_(self.ctx), .valBits = bits })).type_(); + return self.emit(fly.ops.make_copy_atom(self.ctx, &.{}, t, .int(self.ctx, .i32, bits), self.loc())); + } + + /// `op` is an MMA op type, such as a `fly_rocdl.MmaOpCDNA3MFMAType`. + pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { + const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); + return self.emit(fly.ops.make_mma_atom(self.ctx, t, self.loc())); + } + + pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { + const tv = self.toStatic(layout_tv); + const tl = self.toStatic(tile_mn); + const v = self.emit(fly.ops.make_tiled_copy(self.ctx, atom.inner, tv.inner, tl.inner, self.loc())); + return .{ .value = v, .tile = tl }; + } + + /// raked product -> right inverse -> composition -> make_tiled_copy, + /// with the tiler read off the inferred product shape. + pub fn tiledCopyTV(self: *Builder, atom: Value, thr: Layout, val: Layout) TiledCopy { + const thr_size = thr.size() orelse @panic("fly.tiledCopyTV: thr_layout must be static"); + const val_size = val.size() orelse @panic("fly.tiledCopyTV: val_layout must be static"); + const layout_mn = self.static(thr).rakedProduct(self.static(val)); + const tmp = self.static(Layout{ + .shape = .{ .tup = &.{ .static(thr_size), .static(val_size) } }, + .stride = .{ .tup = &.{ .static(1), .static(thr_size) } }, + }); + const layout_tv = layout_mn.rightInverse().composition(tmp); + const tiler = layout_mn.emitShape().productEach(); + // The tiler as a tile literal: one leaf per mode, `[a|b]`. + const shape = tiler.intTupleStatic(); + const tile_modes = self.alloc(Tile, shape.rank()); + for (tile_modes, 0..) |*m, i| m.* = .{ .leaf = switch (shape.at(i)) { + .leaf => |l| l, + else => std.debug.panic("fly.tiledCopyTV: tiler mode {d} is not a leaf", .{i}), + } }; + return self.tiledCopy(atom, layout_tv, Tile{ .modes = tile_modes }); + } + + /// A TiledCopy matched to an MMA operand: the TV layout and tile size + /// come off the `!fly.tiled_mma` type. + pub fn tiledCopyFor(self: *Builder, operand: fly.MmaOperand, copy_atom: Value, tm: TiledMma) TiledCopy { + const tm_ty = fly.expect(tm.value.type_(), .tiled_mma); + const layout_tv = self.staticTy((switch (operand) { + .a => tm_ty.tiledThrValLayoutA(), + .b => tm_ty.tiledThrValLayoutB(), + .c => tm_ty.tiledThrValLayoutC(), + .d => @panic("fly.tiledCopyFor: no D operand copy"), + }).type_()); + const tile_size = self.staticTy(tm_ty.tileSizeMNK().type_()); + const modes: [2]i32 = switch (operand) { + .a => .{ 0, 2 }, + .b => .{ 1, 2 }, + .c => .{ 0, 1 }, + .d => unreachable, + }; + const one = self.intTuple(1); + const tile_modes = self.alloc(Tile, modes.len); + for (modes, 0..) |m, i| { + const lay = self.emit(fly.ops.make_layout(self.ctx, tile_size.selectModes(&.{m}).inner, one.inner, self.loc())); + tile_modes[i] = .{ .layout = self.readLayout(lay.type_()) }; + } + return self.tiledCopy(copy_atom, layout_tv, Tile{ .modes = tile_modes }); + } + + pub fn tiledCopyA(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.a, copy_atom, tm); + } + pub fn tiledCopyB(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.b, copy_atom, tm); + } + pub fn tiledCopyC(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.c, copy_atom, tm); + } + + /// The same tensor over a CDNA buffer resource + /// (`!fly.ptr`), for hardware bounds-checked + /// accesses and `buffer_copy` atoms. Descriptor size is the maximum. + pub fn bufferTensor(self: *Builder, t: Value) Value { + const ptr = t.emitIter(); + const lay = t.emitLayout(); + const c0 = self.constant(.i16, 0); + const nrec = self.constant(.i64, 0xFFFFFFFF); + // (7 << 12) | (4 << 15): the CDNA descriptor flags FlyDSL and XLA use. + const flags = self.constant(.i32, 0x27000); + const elem = t.elemDType(); + const buf_ty = self.ptrType(elem, (must(fly_rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); + const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, .{}, self.loc())); + return buf_ptr.view(lay); + } + + pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { + const al = self.toStatic(atom_layout); + const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, .{ .permutation = if (permutation) |p| p.inner else null }, self.loc())); + return .{ .value = v }; + } + + pub const CopyOpts = struct { + pred: ?Value = null, + }; + + pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { + self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); + } + + pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { + self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, .{}, self.loc())); + } + + // ---------------------------------------------------------------- control flow + + pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); + } + pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); + } + pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); + } + + /// Loop-carried `inits`. Literal bounds take the type of a Value bound, + /// else they are `index`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.constIndex), inits, opts); + } + + /// `value` as a scalar constant of `like`'s element type (`index` for an + /// `index` Value); a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + if (like.type_().isA(mlir.IndexType) != null) { + const i: i128 = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => @intFromFloat(value), + else => value, + }; + return self.constIndex(intBits(i, 64, "index")); + } + return self.constant(like.scalarDType(), value); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ---------------------------------------------------------------- finalization + + /// Append `gpu.return`, verify, and print the module (no debug info: XLA + /// scrubs locations). Owned by the builder's allocator. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + _ = gpu.return_(self.ctx, &.{}, self.loc()).appendTo(self.currentBlock()); + + try common.verify(self.module, "dsl/fly"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; + +// ============================================================================= +// Tests +// ============================================================================= + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +pub fn emitVectorAdd(b: *Builder, thr_layout: Layout, val_layout: Layout, a_in: Value, b_in: Value, c_out: Value) void { + const shape = a_in.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, thr_layout, val_layout); + + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const idC = b.identity(.{ m, n }); + const tile_mn = tc.tileMN(); + + const gA = a_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gB = b_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gC = c_out.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const cC = idC.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + + for (0..@intCast(pC.sizeStatic())) |i| { + pC.set(i, tcC.at(i).elemLess(.{ m, n })); + } + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +test "vectorAdd builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + }); + emitVectorAdd(&b, ordered(.{ 8, 16 }, .{ 1, 0 }), ordered(.{ 1, 4 }, .{ 0, 1 }), a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "module attributes {gpu.container_module}", + "gpu.module @zml_fly_kernels", + "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel", + "fly.static : !fly.layout<(100,1000):(1000,1)>", + "fly.make_copy_atom {valBits = 32 : i32} : !fly.copy_atom, 32>", + "fly.raked_product", + "fly.static : !fly.tile<[8|64]>", + "fly.make_tiled_copy", + "gpu.thread_id x", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.partition_dst", + "fly.make_fragment_like", + "fly.elem_less", + "fly.copy(", + "fly.memref.load_vec", + "arith.addf", + "fly.memref.store_vec", + "gpu.return", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +/// The matrix atom a device family provides. CDNA has MFMA over f32; RDNA has +/// WMMA, whose verifier requires M=N=K=16 and rejects f32 operands, so the +/// operand type and the K tile change with the family, not just the mnemonic. +pub const MmaFlavor = enum { + cdna3_mfma, + gfx11_wmma, + gfx120x_wmma, + + /// MFMA takes f32 operands at 16x16x4; both WMMAs take f16 at 16x16x16. + pub fn atomType(self: MmaFlavor, ctx: *mlir.Context) *const mlir.Type { + const operand = self.operandDType().toMlir(ctx); + const acc: *const mlir.Type = .float(ctx, .f32); + return switch (self) { + .cdna3_mfma => (fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(), + inline .gfx11_wmma, .gfx120x_wmma => |f| blk: { + const T = if (f == .gfx11_wmma) fly_rocdl.MmaOpGFX11WMMAType else fly_rocdl.MmaOpGFX120XWMMAType; + break :blk (T.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(); + }, + }; + } + + /// A and B operand type. The accumulator is f32 either way. + pub fn operandDType(self: MmaFlavor) DType { + return switch (self) { + .cdna3_mfma => .f32, + .gfx11_wmma, .gfx120x_wmma => .f16, + }; + } + + /// Threads one atom occupies: MFMA runs on a wave64, WMMA on a wave32. + /// Checked against the dialect's own `mma_atom.thr_layout` below. + pub fn atomThreads(self: MmaFlavor) i32 { + return switch (self) { + .cdna3_mfma => 64, + .gfx11_wmma, .gfx120x_wmma => 32, + }; + } + + /// `emitTiledMma` lays the atom out (2,2,1), so a block is four of them. + pub fn blockThreads(self: MmaFlavor) i32 { + return 4 * self.atomThreads(); + } + + /// The block tile's K, a multiple of the atom's K. + pub fn blockK(self: MmaFlavor) i64 { + return switch (self) { + .cdna3_mfma => 8, + .gfx11_wmma, .gfx120x_wmma => 16, + }; + } + + /// Buffer copies are CDNA-only: FlyROCDL declares no RDNA copy atom, so + /// RDNA moves through the core universal copy over plain tensors. + fn hasBufferCopy(self: MmaFlavor) bool { + return self == .cdna3_mfma; + } +}; + +/// A two-mode tile from extents known only at emit time. +fn tile2(b: *Builder, x: i64, y: i64) Tile { + const modes = b.alloc(Tile, 2); + modes[0] = .{ .leaf = .{ .s = x } }; + modes[1] = .{ .leaf = .{ .s = y } }; + return .{ .modes = modes }; +} + +/// C = A @ B^T in one block of 256 threads, tiled (2,2,1) over the family's +/// matrix atom. +pub fn emitTiledMma(b: *Builder, flavor: MmaFlavor, a_in: Value, b_in: Value, c_out: Value) void { + const block_m = 64; + const block_n = 64; + const block_k = flavor.blockK(); + + const tid = b.threadId(.x); + const bid = b.blockId(.x); + + const A = if (flavor.hasBufferCopy()) b.bufferTensor(a_in) else a_in; + const B = if (flavor.hasBufferCopy()) b.bufferTensor(b_in) else b_in; + const C = if (flavor.hasBufferCopy()) b.bufferTensor(c_out) else c_out; + + const bA = A.zippedDivide(tile2(b, block_m, block_k)).slice(.{ null, bid }); + const bB = B.zippedDivide(tile2(b, block_n, block_k)).slice(.{ null, bid }); + const bC = C.zippedDivide(tile2(b, block_m, block_n)).slice(.{ null, bid }); + + const mma_atom = b.mmaAtom(flavor.atomType(b.ctx)); + const tiled_mma = b.tiledMma(mma_atom, L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null); + + // A and B move at the operand type, C at the f32 accumulator type. One + // element per instruction: a WMMA fragment's values are strided in memory, + // so a wider copy would move the wrong neighbours. + const copy_ab: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(flavor.operandDType().bitWidth()) }; + const copy_c: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(c_out.elemDType().bitWidth()) }; + const atom_ab = b.copyAtom(copy_ab, flavor.operandDType()); + const atom_c = b.copyAtom(copy_c, c_out.elemDType()); + + const tiled_copy_a = b.tiledCopyA(atom_ab, tiled_mma); + const tiled_copy_b = b.tiledCopyB(atom_ab, tiled_mma); + const tiled_copy_c = b.tiledCopyC(atom_c, tiled_mma); + + const thr_copy_a = tiled_copy_a.getSlice(tid); + const thr_copy_b = tiled_copy_b.getSlice(tid); + const thr_copy_c = tiled_copy_c.getSlice(tid); + + const copy_src_a = thr_copy_a.partitionS(bA); + const copy_src_b = thr_copy_b.partitionS(bB); + const copy_dst_c = thr_copy_c.partitionS(bC); + + const frag_a = tiled_mma.makeFragmentA(bA); + const frag_b = tiled_mma.makeFragmentB(bB); + const frag_c = tiled_mma.makeFragmentC(bC); + + const copy_frag_a = thr_copy_a.retile(frag_a); + const copy_frag_b = thr_copy_b.retile(frag_b); + const copy_frag_c = thr_copy_c.retile(frag_c); + + b.copy(atom_ab, copy_src_a, copy_frag_a, .{}); + b.copy(atom_ab, copy_src_b, copy_frag_b, .{}); + + frag_c.fill(0); + b.gemm(mma_atom, frag_c, frag_a, frag_b, frag_c); + + b.copy(atom_c, copy_frag_c, copy_dst_c, .{}); +} + +test "tiledMma builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .cdna3_mfma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "!fly.ptr", + "fly.make_mma_atom : !fly.mma_atom f32>>", + "fly.make_tiled_mma", + "fly.static : !fly.int_tuple<(32,32,4)>", + "fly.mma.make_fragment", + "fly.tiled_copy.retile", + "vector.broadcast", + "fly.gemm(", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "MmaFlavor.atomThreads matches the dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (comptime std.enums.values(MmaFlavor)) |flavor| { + const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); + // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch + // geometry is derived from it, so a wrong table silently corrupts C. + const threads = switch (atom.thrLayout().shape().leaf()) { + .static => |v| v, + else => return error.TestUnexpectedResult, + }; + try std.testing.expectEqual(@as(i64, flavor.atomThreads()), threads); + } +} + +test "tiledMma builds for the RDNA WMMA atom" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma_wmma"); + defer b.deinit(); + const k = MmaFlavor.gfx11_wmma.blockK(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .b = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .gfx11_wmma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // No buffer descriptor: FlyROCDL declares no RDNA copy atom, so this path + // moves through the core universal copy over plain tensors. + if (std.mem.indexOf(u8, ir, "buffer_desc") != null) { + std.debug.print("unexpected buffer descriptor on the WMMA path:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } + for ([_][]const u8{ + "!fly_rocdl.gfx11.wmma<16x16x16, (f16, f16) -> f32", + "!fly.copy_atom, 16>", + "fly.gemm(", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "scalar conversions and constants" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f16, .dims = &.{16} } }, + .p = .{ .tensor = .{ .dtype = .i1, .dims = &.{16} } }, + }); + + // A predicate is unsigned: `true` must convert to 1, never -1. + const pred = b.constant(.i32, 3).lt(7); + _ = pred.to(.f32); + _ = pred.to(.i32); + // Equal-width float pairs are neither ext nor trunc. + _ = b.constant(.f16, 1.0).to(.bf16); + // A float constant outside i64 range must not go through @intFromFloat. + _ = b.constant(.f32, -std.math.inf(f32)); + _ = b.constant(.f32, std.math.floatMax(f32)); + // A `.bool` argument keeps its storage element type end to end. + try std.testing.expectEqual(DType.i8, a.p.elemDType()); + try std.testing.expectEqual(DType.f16, a.x.elemDType()); + // Literal inits are lifted, and an integer constant may be a bit pattern. + var loop = b.openFor(0, 4, 1, .{ 0, 0.5 }); + { + loop.yield(.{ loop.carried[0].bitXor(b.constant(.i32, 0xd4d0ccc8)), loop.carried[1].mul(2.0) }); + } + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "arith.uitofp", + "arith.extui", + "arith.convertf", + "arith.constant 0xFF800000 : f32", + "!fly.memref", + "-> (i32, f32)", + "arith.constant -724513592 : i32", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + if (std.mem.indexOf(u8, ir, "arith.sitofp") != null or std.mem.indexOf(u8, ir, "arith.extsi") != null) { + std.debug.print("i1 was sign-extended:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } +} + +test "shuffles and fast math carry their attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "shuffle"); + defer b.deinit(); + _ = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .dims = &.{16} } }, + }); + + b.setFastMath(true); + const x = b.constant(.f32, 1.0); + _ = x.shuffleXor(1, 64).add(x.shuffle(.idx, 3, 64)); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "gpu.shuffle xor", + "gpu.shuffle idx", + "arith.addf %shuffleResult, %shuffleResult_1 fastmath", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } +} + +test "static queries read inferred types" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "q"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .tensor = .{ .dtype = .bf16, .dims = &.{ 64, 32 } } } }); + try std.testing.expectEqual(@as(i64, 64 * 32), a.x.sizeStatic()); + try std.testing.expectEqual(DType.bf16, a.x.elemDType()); + try std.testing.expectEqual(Value.Kind.memref, a.x.kind()); + const tiled = a.x.flatDivide(tile(.{ 16, 8 })); + try std.testing.expectEqual(@as(i64, 64 * 32), tiled.sizeStatic()); + try std.testing.expectEqual(@as(usize, 4), tiled.shapeStatic().rank()); +} + +test "atomic addition builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "atomic_add"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const counter = b.sharedArray(.i32, 1, 4); + _ = b.ptrAtomicAdd(counter.emitIter(), b.constant(.i32, 1), .workgroup); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // LLVM's current custom assembly stores alignment in the property dictionary. + for ([_][]const u8{ + "llvm.atomicrmw add", + "syncscope(\"workgroup\") monotonic ", + ": !llvm.ptr<3>, i32", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("IR missing `{s}`:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "SSA matrix atom builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "mma_atom_ssa"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const atom_type = try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .bf16), + .elemTyB = .float(ctx, .bf16), + .elemTyAcc = .float(ctx, .f32), + }); + const atom = b.mmaAtom(atom_type.type_()); + const lhs = b.constant(.bf16, 0).splat(4); + const rhs = b.constant(.bf16, 0).splat(4); + const acc = b.constant(.f32, 0).splat(4); + const result = b.mmaAtomCall(atom, lhs, rhs, acc); + try std.testing.expect(result.type_().eql(acc.type_())); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "comparisons take index ivs and vectors" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "compare"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + // Literal bounds make an `index` loop. + var loop = b.openFor(0, 4, 1, .{}); + { + _ = loop.iv.lt(2); + loop.close(); + } + // A literal or a scalar broadcasts against a vector. + const v = b.constant(.f32, 1.0).splat(4); + _ = v.lt(0.5); + _ = v.ge(b.constant(.f32, 2.0)); + _ = b.constant(.i32, 3).splat(4).eq(3); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][2][]const u8{ + .{ "arith.cmpi slt", ": index" }, + .{ "arith.cmpf olt", ": vector<4xf32>" }, + .{ "arith.cmpf oge", ": vector<4xf32>" }, + .{ "arith.cmpi eq", ": vector<4xi32>" }, + }) |needles| { + var lines = std.mem.splitScalar(u8, ir, '\n'); + const found = while (lines.next()) |line| { + if (std.mem.indexOf(u8, line, needles[0]) != null and std.mem.endsWith(u8, line, needles[1])) break true; + } else false; + if (!found) { + std.debug.print("no `{s} ... {s}` in:\n{s}\n", .{ needles[0], needles[1], ir }); + return error.TestUnexpectedResult; + } + } +} diff --git a/dsl/fly/dtype.zig b/dsl/fly/dtype.zig new file mode 100644 index 00000000..ecc9f070 --- /dev/null +++ b/dsl/fly/dtype.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + }; + } + + pub fn bitWidth(self: DType) u32 { + return switch (self) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => true, + else => false, + }; + } + + /// The element type a kernel argument is stored with on device: + /// predicates live in one byte. A copy atom over `!fly.ptr` would + /// claim `bits / 1` elements while moving `bits / 8` bytes, so an + /// argument keeps its storage type and the body converts explicitly. + pub fn storageElem(self: DType) DType { + return switch (self) { + .i1 => .i8, + else => self, + }; + } +}; + +/// Inverse of `toMlir`. +pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { + inline for (comptime std.enums.values(DType)) |dt| { + if (ty.eql(dt.toMlir(ctx))) return dt; + } + return null; +} + +test "dtype types round-trip" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (comptime std.enums.values(DType)) |dt| { + try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); + } + try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); + try std.testing.expectEqual(DType.f8e4m3fnuz, DType.f8e4m3fnuz.storageElem()); +} diff --git a/dsl/fly/layout.zig b/dsl/fly/layout.zig new file mode 100644 index 00000000..06dd64c6 --- /dev/null +++ b/dsl/fly/layout.zig @@ -0,0 +1,544 @@ +const std = @import("std"); + +const fly = @import("mlir/dialects/fly"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const attributes = fly.attributes; + +/// Enough for any tuple or tile a kernel writes by hand. +const max_modes = 32; + +/// A dynamic integer leaf: `?`, `?{i64}`, `?{div=8}`, `?{i64 div=8}`. +pub const Dyn = struct { + width: u8 = 32, + div: u32 = 1, +}; + +/// Static, dynamic, or the `*` wildcard. +pub const Leaf = union(enum) { + s: i64, + d: Dyn, + none, + + pub fn print(self: Leaf, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .s => |v| try w.print("{d}", .{v}), + .none => try w.writeAll("*"), + .d => |dy| { + try w.writeAll("?"); + if (dy.width != 32 or dy.div != 1) { + try w.writeAll("{"); + if (dy.width != 32) try w.print("i{d}", .{dy.width}); + if (dy.div != 1) { + if (dy.width != 32) try w.writeAll(" "); + try w.print("div={d}", .{dy.div}); + } + try w.writeAll("}"); + } + }, + } + } +}; + +fn leafAttr(leaf: Leaf, ctx: *mlir.Context) mlir.Error!*const attributes.IntAttr { + return switch (leaf) { + .s => |v| .getStatic(ctx, @intCast(v)), + .d => |dy| .getDynamic(ctx, dy.width, @intCast(dy.div)), + .none => .getNone(ctx), + }; +} + +/// A scaled basis element: `vE0`, `2E1E2`. +pub const Basis = struct { + value: Leaf, + modes: []const i32, +}; + +pub const IntTuple = union(enum) { + leaf: Leaf, + basis: Basis, + tup: []const IntTuple, + + pub const dyn: IntTuple = .{ .leaf = .{ .d = .{} } }; + pub const star: IntTuple = .{ .leaf = .none }; + + pub fn static(v: i64) IntTuple { + return .{ .leaf = .{ .s = v } }; + } + + pub fn dynamic(width: u8, div: u32) IntTuple { + return .{ .leaf = .{ .d = .{ .width = width, .div = div } } }; + } + + /// 1 for a leaf. + pub fn rank(self: IntTuple) usize { + return switch (self) { + .tup => |t| t.len, + else => 1, + }; + } + + pub fn isLeaf(self: IntTuple) bool { + return self != .tup; + } + + pub fn at(self: IntTuple, i: usize) IntTuple { + return switch (self) { + .tup => |t| t[i], + else => if (i == 0) self else @panic("fly.IntTuple.at: index out of range on a leaf"), + }; + } + + /// Null if any leaf is dynamic or `*`. + pub fn product(self: IntTuple) ?i64 { + switch (self) { + .leaf => |l| return switch (l) { + .s => |v| v, + else => null, + }, + .basis => return null, + .tup => |t| { + var p: i64 = 1; + for (t) |e| p *= e.product() orelse return null; + return p; + }, + } + } + + pub fn isStatic(self: IntTuple) bool { + return switch (self) { + .leaf => |l| l == .s, + .basis => |b| b.value == .s, + .tup => |t| for (t) |e| { + if (!e.isStatic()) break false; + } else true, + }; + } + + /// At any depth. + pub fn leafCount(self: IntTuple) usize { + return switch (self) { + .tup => |t| blk: { + var n: usize = 0; + for (t) |e| n += e.leafCount(); + break :blk n; + }, + else => 1, + }; + } + + pub fn print(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .basis => |b| { + try b.value.print(w); + for (b.modes) |m| try w.print("E{d}", .{m}); + }, + .tup => |t| { + try w.writeAll("("); + for (t, 0..) |e, i| { + if (i > 0) try w.writeAll(","); + try e.print(w); + } + try w.writeAll(")"); + }, + } + } + + pub fn format(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: IntTuple, ctx: *mlir.Context) mlir.Error!*const attributes.IntTupleAttr { + return switch (self) { + .leaf => |l| .get(ctx, .{ .value = (try leafAttr(l, ctx)).attribute() }), + .basis => |b| .getBasis(ctx, try leafAttr(b.value, ctx), b.modes), + .tup => |elems| blk: { + var buf: stdx.BoundedArray(*const attributes.IntTupleAttr, max_modes) = .empty; + for (elems) |e| buf.appendAssumeCapacity(try e.toAttr(ctx)); + break :blk .getTuple(ctx, buf.constSlice()); + }, + }; + } + + pub fn eql(a: IntTuple, b: IntTuple) bool { + if (std.meta.activeTag(a) != std.meta.activeTag(b)) return false; + return switch (a) { + .leaf => |l| std.meta.eql(l, b.leaf), + .basis => |x| std.meta.eql(x.value, b.basis.value) and std.mem.eql(i32, x.modes, b.basis.modes), + .tup => |t| t.len == b.tup.len and for (t, b.tup) |x, y| { + if (!x.eql(y)) break false; + } else true, + }; + } +}; + +/// Integers become static leaves, tuples nest, `null` is `*`, an `IntTuple` +/// passes through: `it(.{ .{ 2, 4 }, 8 })` is `((2,4),8)`. +pub fn it(comptime v: anytype) IntTuple { + return comptime itImpl(v); +} + +fn itImpl(comptime v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == IntTuple) return v; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return IntTuple.star; + switch (@typeInfo(T)) { + .comptime_int, .int => return IntTuple.static(v), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); + var elems: [s.field_names.len]IntTuple = undefined; + for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); + const frozen = elems; + return .{ .tup = &frozen }; + }, + else => @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)), + } +} + +/// `!fly.layout`. +pub const Layout = struct { + shape: IntTuple, + stride: IntTuple, + + pub fn rank(self: Layout) usize { + return self.shape.rank(); + } + + /// Null if dynamic. + pub fn size(self: Layout) ?i64 { + return self.shape.product(); + } + + pub fn print(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.shape.print(w); + try w.writeAll(":"); + try self.stride.print(w); + } + + pub fn format(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Layout, ctx: *mlir.Context) mlir.Error!*const attributes.LayoutAttr { + return .get(ctx, .{ .shape = try self.shape.toAttr(ctx), .stride = try self.stride.toAttr(ctx) }); + } + + pub fn eql(a: Layout, b: Layout) bool { + return a.shape.eql(b.shape) and a.stride.eql(b.stride); + } +}; + +/// Two congruent tuples: `L(.{ 4, 8 }, .{ 1, 4 })` is `(4,8):(1,4)`. +pub fn L(comptime shape: anytype, comptime stride: anytype) Layout { + return comptime blk: { + const s = it(shape); + const d = it(stride); + checkCongruent(s, d); + break :blk .{ .shape = s, .stride = d }; + }; +} + +fn checkCongruent(comptime a: IntTuple, comptime b: IntTuple) void { + if (a.isLeaf() != b.isLeaf() or a.rank() != b.rank()) { + @compileError("fly.L: shape and stride are not congruent"); + } + if (a == .tup) { + for (a.tup, b.tup) |x, y| checkCongruent(x, y); + } +} + +/// Last mode contiguous: `rowMajor(.{ 4, 8 })` is `(4,8):(8,1)`. +pub fn rowMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(n - 1 - i); + break :blk orderedFlat(s, &order); + }; +} + +/// First mode contiguous. +pub fn colMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(i); + break :blk orderedFlat(s, &order); + }; +} + +/// `make_ordered_layout` folded at comptime: the smallest order value gets +/// stride 1, the next the product of the previous extents, and so on. +/// `ordered(.{ 8, 16 }, .{ 1, 0 })` is `(8,16):(16,1)`. +pub fn ordered(comptime shape: anytype, comptime order: anytype) Layout { + return comptime blk: { + const s = it(shape); + const o = it(order); + checkCongruent(s, o); + const n = s.leafCount(); + var flat_order: [n]i32 = undefined; + var idx: usize = 0; + flattenOrder(o, &flat_order, &idx); + break :blk orderedFlat(s, &flat_order); + }; +} + +fn flattenOrder(comptime o: IntTuple, out: []i32, idx: *usize) void { + switch (o) { + .leaf => |l| { + out[idx.*] = @intCast(l.s); + idx.* += 1; + }, + .tup => |t| for (t) |e| flattenOrder(e, out, idx), + .basis => @compileError("fly.ordered: basis elements are not allowed in an order"), + } +} + +fn orderedFlat(comptime shape: IntTuple, comptime order: []const i32) Layout { + comptime { + const n = shape.leafCount(); + var extents: [n]i64 = undefined; + var idx: usize = 0; + flattenStatic(shape, &extents, &idx); + var strides: [n]i64 = undefined; + var assigned: [n]bool = @splat(false); + var acc: i64 = 1; + for (0..n) |_| { + var best: ?usize = null; + for (0..n) |i| { + if (assigned[i]) continue; + if (best == null or order[i] < order[best.?]) best = i; + } + strides[best.?] = acc; + acc *= extents[best.?]; + assigned[best.?] = true; + } + var pos: usize = 0; + return .{ .shape = shape, .stride = unflatten(shape, &strides, &pos) }; + } +} + +fn flattenStatic(comptime t: IntTuple, out: []i64, idx: *usize) void { + switch (t) { + .leaf => |l| switch (l) { + .s => |v| { + out[idx.*] = v; + idx.* += 1; + }, + else => @compileError("fly: a folded layout needs a fully static shape"), + }, + .basis => @compileError("fly: a folded layout needs a plain integer shape"), + .tup => |elems| for (elems) |e| flattenStatic(e, out, idx), + } +} + +fn unflatten(comptime like: IntTuple, values: []const i64, pos: *usize) IntTuple { + switch (like) { + .tup => |elems| { + var out: [elems.len]IntTuple = undefined; + for (elems, 0..) |e, i| out[i] = unflatten(e, values, pos); + const frozen = out; + return .{ .tup = &frozen }; + }, + else => { + const v = values[pos.*]; + pos.* += 1; + return IntTuple.static(v); + }, + } +} + +/// A leaf (`8` or `*`), a layout, or a nested tile. +pub const Tile = union(enum) { + leaf: Leaf, + layout: Layout, + modes: []const Tile, + + pub fn print(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .layout => |l| try l.print(w), + .modes => |m| { + try w.writeAll("["); + for (m, 0..) |e, i| { + if (i > 0) try w.writeAll("|"); + try e.print(w); + } + try w.writeAll("]"); + }, + } + } + + pub fn format(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + /// One mode: an int, a layout, or a nested tile. + fn modeAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const mlir.Attribute { + return switch (self) { + .leaf => |l| (try leafAttr(l, ctx)).attribute(), + .layout => |l| (try l.toAttr(ctx)).attribute(), + .modes => (try self.toAttr(ctx)).attribute(), + }; + } + + pub fn toAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const attributes.TileAttr { + const modes = switch (self) { + .modes => |m| m, + else => return .get(ctx, .{ .value = try self.modeAttr(ctx) }), + }; + var buf: stdx.BoundedArray(*const mlir.Attribute, max_modes) = .empty; + for (modes) |m| buf.appendAssumeCapacity(try m.modeAttr(ctx)); + return .getModes(ctx, buf.constSlice()); + } +}; + +/// `tile(.{ 128, 64 })` is `[128|64]`; `tile(.{ null, 8 })` is `[*|8]`. +pub fn tile(comptime v: anytype) Tile { + return comptime tileImpl(v); +} + +fn tileImpl(comptime v: anytype) Tile { + const T = @TypeOf(v); + if (T == Tile) return v; + if (T == Layout) return .{ .layout = v }; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return .{ .leaf = .none }; + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .leaf = .{ .s = v } }, + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); + var elems: [s.field_names.len]Tile = undefined; + for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); + const frozen = elems; + return .{ .modes = &frozen }; + }, + else => @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)), + } +} + +/// `S`. +pub const Swizzle = struct { + mask: i32, + base: i32, + shift: i32, + + pub const trivial: Swizzle = .{ .mask = 0, .base = 0, .shift = 0 }; + + pub fn isTrivial(self: Swizzle) bool { + return self.mask == 0 and self.base == 0 and self.shift == 0; + } + + pub fn print(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.print("S<{d},{d},{d}>", .{ self.mask, self.base, self.shift }); + } + + pub fn format(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Swizzle, ctx: *mlir.Context) mlir.Error!*const attributes.SwizzleAttr { + return .get(ctx, .{ .mask = self.mask, .base = self.base, .shift = self.shift }); + } +}; + +// ----------------------------------------------------------------------------- +// Tests +// ----------------------------------------------------------------------------- + +fn expectPrints(expected: []const u8, v: anytype) !void { + var buf: [1024]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{v}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "int tuple literals print in fly syntax" { + try expectPrints("42", it(42)); + try expectPrints("((2,4),8)", it(.{ .{ 2, 4 }, 8 })); + try expectPrints("(?,8)", it(.{ IntTuple.dyn, 8 })); + try expectPrints("?{i64 div=8}", IntTuple.dynamic(64, 8)); + try expectPrints("?{div=4}", IntTuple.dynamic(32, 4)); + try expectPrints("?{i64}", IntTuple.dynamic(64, 1)); + try expectPrints("(*,8)", it(.{ null, 8 })); + try expectPrints("(1,(2,3))", it(.{ 1, .{ 2, 3 } })); +} + +test "layout literals" { + try expectPrints("(4,8):(1,4)", L(.{ 4, 8 }, .{ 1, 4 })); + try expectPrints("((2,4),8):((1,2),8)", L(.{ .{ 2, 4 }, 8 }, .{ .{ 1, 2 }, 8 })); + try expectPrints("(4,8):(8,1)", rowMajor(.{ 4, 8 })); + try expectPrints("(4,8):(1,4)", colMajor(.{ 4, 8 })); + try expectPrints("((2,4),8):((32,8),1)", rowMajor(.{ .{ 2, 4 }, 8 })); + try expectPrints("(8,16):(16,1)", ordered(.{ 8, 16 }, .{ 1, 0 })); + try expectPrints("(1,4):(1,1)", ordered(.{ 1, 4 }, .{ 0, 1 })); + try expectPrints("(4,2,8):(2,1,8)", ordered(.{ 4, 2, 8 }, .{ 1, 0, 2 })); +} + +test "tiles and swizzles" { + try expectPrints("[128|64]", tile(.{ 128, 64 })); + try expectPrints("[*|8]", tile(.{ null, 8 })); + try expectPrints("[(4,8):(1,4)|16]", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectPrints("[[2|3]|4]", tile(.{ .{ 2, 3 }, 4 })); + try expectPrints("S<3,3,3>", Swizzle{ .mask = 3, .base = 3, .shift = 3 }); +} + +test "static queries" { + try std.testing.expectEqual(@as(?i64, 64), it(.{ .{ 2, 4 }, 8 }).product()); + try std.testing.expectEqual(@as(?i64, null), it(.{ IntTuple.dyn, 8 }).product()); + try std.testing.expectEqual(@as(usize, 2), it(.{ .{ 2, 4 }, 8 }).rank()); + try std.testing.expectEqual(@as(usize, 3), it(.{ .{ 2, 4 }, 8 }).leafCount()); + try std.testing.expect(it(.{ 2, 4 }).eql(it(.{ 2, 4 }))); + try std.testing.expect(!it(.{ 2, 4 }).eql(it(.{ 4, 2 }))); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + @import("builder.zig").registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// The type built from the host model must be the type the printed literal +/// parses to: the two paths may never drift. +fn expectBuilt(ctx: *mlir.Context, comptime T: type, comptime fmt: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print(fmt, .{value}); + const parsed = try mlir.Type.parse(ctx, w.buffered()); + const built = try T.get(ctx, .{ .attr = try value.toAttr(ctx) }); + if (!parsed.eql(built.type_())) { + std.debug.print("parsed `{f}` but built `{f}`\n", .{ parsed, built }); + return error.TestUnexpectedResult; + } +} + +test "the host model builds the types its literals print" { + const ctx = try testContext(); + defer ctx.deinit(); + const IntTupleType = fly.types.IntTupleType; + const LayoutType = fly.types.LayoutType; + const TileType = fly.types.TileType; + + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(42)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ .{ 2, 4 }, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ IntTuple.dyn, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple.dynamic(64, 8)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ null, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple{ .basis = .{ .value = .{ .s = 2 }, .modes = &.{ 1, 2 } } }); + + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", L(.{ 4, 8 }, .{ 1, 4 })); + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", rowMajor(.{ .{ 2, 4 }, 8 })); + + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ 128, 64 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ null, 8 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ .{ 2, 3 }, 4 })); +} diff --git a/dsl/metal/BUILD.bazel b/dsl/metal/BUILD.bazel new file mode 100644 index 00000000..d68355c0 --- /dev/null +++ b/dsl/metal/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = ["types.zig"], + import_name = "dsl/metal/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/air", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/vector", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/metal/README.md b/dsl/metal/README.md new file mode 100644 index 00000000..1cbc40ce --- /dev/null +++ b/dsl/metal/README.md @@ -0,0 +1,150 @@ +# ZML Metal DSL + +A Zig builder for Metal kernels that reads like MSL. It emits the `air` +dialect (`mlir/dialects/air`) and the parts of `arith`, `cf`, `vector` and +`llvm` that MSL code compiles to: values, locals in memory, blocks and +branches. `zml.kernel.metal.Kernel` wraps a kernel into +a `__gpu$xla.gpu.air` custom call, and XLA does the rest: it lowers the module to +Apple's AIR, runs the Metal frontend's optimization pipeline +(`air-opt -O2 -disable-loop-unrolling`) and builds a metallib. Apple's +backend compiles that metallib the way it compiles an MSL kernel. + +## Skeleton of a kernel + +```zig +const metal = @import("zml").kernel.metal; + +pub const Axpy = struct { + pub const Cfg = struct {}; + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +// In a model: +const out = Axpy.Kernel.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ 4, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, 1000)) }, +}).out; +``` + +`declareArgs` declares the kernel parameters in order, which is the `[[buffer(n)]]` +order. `inputs` and `outputs` bind buffers to tensors by name. An output named +like an input updates that input's buffer in place. `constants` bind +`constant T &` arguments by value. `Constant.of` takes a scalar, or a host +`extern struct` laid out like the MSL struct. `CallOpts.threadgroup_memory` +sizes `[[threadgroup(0)]]`. + +Function constants have no runtime form. A kernel takes them through its `Cfg`, +so each configuration emits its own specialized kernel. + +## Arguments + +| MSL | `ArgSpec` | +|---|---| +| `device T *p`, `const device T *p` | `.buffer = .{ .type = T, .const_ = true }` | +| `constant T &x` | `.constant = T`, read wherever it is used, as C++ reads a reference | +| `constant S &s` | `.constant_struct = S`, read with `s.field("name")` | +| `threadgroup T *p [[threadgroup(0)]]` | `.threadgroup = T` | +| `uint3 tid [[thread_position_in_grid]]` and the other builtins | `.thread_position_in_grid = .uint3`, ... | + +## Values + +A `Value` is a scalar or vector (`Type`, spelled as in MSL: `.float4`, +`.ushort`), a pointer, a private array, a `simdgroup_matrix` or a constant +struct. Operations follow C++: integer promotion and the usual arithmetic +conversions pick the result type, signed arithmetic is `nsw`, and float math +carries the kernel's fast-math flags. A Zig number is the MSL literal it +spells, an `int` (a `long` when it does not fit) or a `float`, and a runtime +Zig integer keeps its width and signedness; the usual conversions follow, so +`h.mul(0.5)` on a `half` is a `float` multiply, as `h * 0.5` is in MSL. A +scalar next to a vector takes its element type. Stores, assignments and the +functions of one type (`max`, `min`, `clamp`, `fma`) convert a number to their +type. `0.5h` is `b.constant(.half, 0.5)`. + +| MSL | DSL | +|---|---| +| `a + b`, `a * b`, `a < b`, `a & b` | `a.add(b)`, `a.mul(b)`, `a.lt(b)`, `a.bitAnd(b)` | +| `(float) x`, `float4(v)` | `x.to(.float)`, `v.to(.float4)` | +| `as_type(x)` | `x.asType(.uint)` | +| `v.x`, `v[i]`, `v[i] = e`, `v.y = e` | `v.x()`, `v.at(i)`, `v = v.insert(i, e)`, `v = v.insert(1, e)`: a subscript takes the index's type (a literal is an `int`) | +| `p[i]`, `p[i] = v`, `p + i` | `p.load(i)`, `p.store(i, v)`, `p.add(i)` | +| `(threadgroup float4 *) p` | `p.to(.float4)` | +| `T x;`, `T x = e;`, `x = e` | `const x = b.local(.T, null)`, `b.local(.T, e)`, `x.assign(e)` | +| `device T *p = e;` that the code moves | `const p = b.localPointer(e)`, `p.assign(p.add(i))` | +| `float s[8];`, `float s[8] = {};` | `const s = b.array(.float, 8)`, `b.zeroedArray(.float, 8)`, then `s.load(i)` and `s.store(i, v)` | +| `&x` of a local | `x.address()` | +| `float4(a, b, c, d)`, `1.0f` | `b.vec(.float4, .{ a, b, c, d })`, `b.constant(.float, 1)` | +| `c ? x : y` | `b.ternary(c, x, y)`, a select of evaluated operands; `var t = b.openIfElse(c, .{.T})`, `x`, `t.yieldThen(.{x})`, `y`, `t.yieldElse(.{y})`, `t.results[0]` | +| `exp(x)`, `fast::exp(x)`, `precise::tanh(x)` | `b.exp(x)`, `b.fast().exp(x)`, `b.precise().tanh(x)` | +| `fma`, `dot`, `max`, `clamp` | `b.fma`, `b.dot`, `b.max`, `b.clamp` | +| `simd_sum(x)`, `simd_shuffle_xor(x, m)` | `b.simdSum(x)`, `b.simdShuffleXor(x, m)` | +| `threadgroup_barrier(mem_flags::mem_threadgroup)` | `b.threadgroupBarrier(.mem_threadgroup)` | +| `simdgroup_load(m, p, ld, origin, transpose)`, `simdgroup_store(m, p, ld)` | `const m = b.simdgroupLoad(p, ld, .{ .origin = ..., .transpose = true })`, `b.simdgroupStore(m, p, ld, .{})` | +| `simdgroup_multiply_accumulate(d, a, b, c)` | `const d = b.simdgroupMultiplyAccumulate(a, b, c)` | +| `m.thread_elements()` read, assigned | `m.threadElements()`; `const m = b.matrix(.T, elements)` | + +## Control flow + +| MSL | DSL | +|---|---| +| `for (T i = lower; i < upper; i += s)`, `++i` | `var l = b.openFor(lower, upper, s)`, `l.iv`, `l.close()`; a step of 1 is `++i` | +| `#pragma unroll`, `#pragma clang loop unroll(full)` | `b.openForOpts(..., .{ .unroll = .enable })`, `.{ .unroll = .full }` | +| `while (c) { ... }`, `for (init; c; inc)`, `for (;;)` | `var l = b.openLoop(.{})`, `l.body(c)`, the body, `l.next()`, the increment, `l.close()` | +| `break;` | `l.breakLoop()` | +| `if (c) { ... }` | `var s = b.openIf(c)`, `s.close()` | +| `if (c) { ... } else { ... }` | `var s = b.openIfElse(c, .{})`, `s.elseBranch()`, `s.close()` | +| `if (a \|\| (b && !x))` | `var c = b.condition()`, `c.or_(a)`, `c.and_(b)`, `var s = c.openIfNot(x)` | +| `x = a && b && c` | `var l = b.openAnd()`, `l.term(a)`, `l.term(b)`, `l.close(c)` | +| `return;` | `b.return_()`, or `b.returnIf(c)` | + +`condition`, `openAnd`, `openOr` and a ternary `openIfElse` skip a term or an +operand, so use them only when one must not run, such as a load past the end +of a buffer. Combine comparisons with `bitAnd` and `bitOr`, and pick between +values that are safe to compute with `b.ternary`. + +Put each `if` or loop body in a Zig `{ ... }` block right after the opener, +one block per region, each ending with the call that ends the region: +`elseBranch()` or `yieldThen`, then `close()` or `yieldElse` for an `if`; +`body(c)`, `next()`, then `close()` for `openLoop`. The scope variable stays +outside, so its `results` stay readable after the blocks. + +A value is evaluated where it is written; a Zig `const` holds it, and an MSL +function is a Zig function that emits its body. A value a loop or branch +changes is a local, which lives in memory until the pipeline promotes it. +The terms of `&&` and `||` run only when the ones before leave the result +open, and an `openIfElse(c, .{T})` operand only when `c` selects it, each in +its own block. Loops follow C++: every +loop is `mustprogress`, and its condition and increment follow C++'s +conversions, so a `short` loop variable compares as an `int` and `++i` steps +it as a `short`. + +A `simdgroup_matrix` is a value of its 64 elements, of which a lane holds the +first two; the load, store and multiply-accumulate call the +`air.simdgroup_matrix_8x8_*` builtins. + +## Checking a port + +Fast math lets Apple's backend reassociate and contract float math, so a port +computes what its MSL kernel computes up to rounding, not always to the bit. +Compare its outputs with the MSL kernel's on the same inputs, and time both +with the same grid, threadgroup size and threadgroup memory. diff --git a/dsl/metal/builder.zig b/dsl/metal/builder.zig new file mode 100644 index 00000000..dea3d48a --- /dev/null +++ b/dsl/metal/builder.zig @@ -0,0 +1,2320 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +pub const types = @import("types.zig"); + +pub const Scalar = types.Scalar; +pub const Type = types.Type; +pub const Struct = types.Struct; +pub const StructField = types.StructField; +pub const Builtin = air.Builtin; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ air, arith, cf, func, llvm, vector }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const AddressSpace = air.AddressSpace; + +/// `mem_flags`: `.mem_threadgroup`, or `.{ .device = true, .threadgroup = true }` +/// for `mem_device | mem_threadgroup`. +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const mem_none: MemFlags = .{}; + pub const mem_device: MemFlags = .{ .device = true }; + pub const mem_threadgroup: MemFlags = .{ .threadgroup = true }; + pub const mem_texture: MemFlags = .{ .texture = true }; +}; + +/// A kernel argument, as `Builder.declareArgs` takes it. Device and constant +/// arguments take the next `[[buffer(n)]]`, threadgroup arguments the next +/// `[[threadgroup(n)]]`, in declaration order. +pub const ArgSpec = union(enum) { + /// `device T *p`, or `const device T *p` with `.const_`. + buffer: Buffer, + /// `constant T &x`; the argument is its value. + constant: Type, + /// `constant S &s`; `Value.field` reads a field. + constant_struct: Struct, + /// `threadgroup T *p`. + threadgroup: Type, + thread_position_in_grid: Type, + threadgroup_position_in_grid: Type, + thread_position_in_threadgroup: Type, + threads_per_threadgroup: Type, + threadgroups_per_grid: Type, + threads_per_grid: Type, + thread_index_in_simdgroup: Type, + simdgroup_index_in_threadgroup: Type, + threads_per_simdgroup: Type, + simdgroups_per_threadgroup: Type, + thread_index_in_threadgroup: Type, + + pub const Buffer = struct { + type: Type, + const_: bool = false, + /// `__restrict__`. + restrict: bool = false, + }; +}; + +/// `raw` as an `ArgSpec`: an `ArgSpec`, or a literal of one of its variants +/// such as `.{ .buffer = .{ .type = .float } }`. +fn argSpec(raw: anytype) ArgSpec { + if (@TypeOf(raw) == ArgSpec) return raw; + const variant = comptime literalVariant(@TypeOf(raw), "ArgSpec"); + const inner = @field(raw, variant); + return switch (@field(std.meta.Tag(ArgSpec), variant)) { + .buffer => blk: { + comptime knownFields(@TypeOf(inner), ArgSpec.Buffer, "ArgSpec.buffer"); + break :blk .{ .buffer = .{ + .type = typeOf(inner.type), + .const_ = if (@hasField(@TypeOf(inner), "const_")) inner.const_ else false, + .restrict = if (@hasField(@TypeOf(inner), "restrict")) inner.restrict else false, + } }; + }, + .constant_struct => .{ .constant_struct = inner }, + inline else => |tag| @unionInit(ArgSpec, @tagName(tag), typeOf(inner)), + }; +} + +/// The variant a spec literal names: one, as a union literal does. +fn literalVariant(Literal: type, what: []const u8) []const u8 { + const names = @typeInfo(Literal).@"struct".field_names; + if (names.len != 1) @compileError("dsl/metal: " ++ what ++ " literals name one variant, not " ++ std.fmt.comptimePrint("{d}", .{names.len})); + return names[0]; +} + +/// Rejects a field of a spec literal that `Options` does not have. +fn knownFields(Literal: type, Options: type, what: []const u8) void { + for (@typeInfo(Literal).@"struct".field_names) |name| { + if (!@hasField(Options, name)) @compileError("dsl/metal: " ++ what ++ " has no field " ++ name); + } +} + +/// A `Type`, or the name of one (`.float`). +fn typeOf(t: anytype) Type { + return if (@TypeOf(t) == @EnumLiteral()) @field(Type, @tagName(t)) else t; +} + +/// A declared device, constant or threadgroup argument, for binding. +pub const Argument = struct { + name: []const u8, + kind: enum { buffer, constant, threadgroup }, + /// The `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + index: u32, + /// The size of a constant argument. + size: u64 = 0, +}; + +/// A pointer: its address space and pointee type. +pub const Pointer = struct { + space: AddressSpace, + pointee: Type, +}; + +/// An SSA value with its MSL type. Operators follow C++: operands take the +/// usual arithmetic conversions, integers narrower than `int` are promoted, +/// and signedness picks the operation (`uint / int` divides unsigned). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + kind: Kind = undefined, + + pub const Kind = union(enum) { + value: Type, + pointer: Pointer, + /// A `constant S &` argument. + constant_struct: Struct, + /// A `constant T &` argument or a local: an lvalue, read where it is + /// used as C++ reads it. + reference: Pointer, + /// A `simdgroup_matrix`: its 64 elements, of which a lane + /// holds the first two. + matrix: Scalar, + /// A pointer local. + pointer_reference: Pointer, + }; + + /// The value itself, an lvalue read. + pub fn rvalue(self: Value) Value { + return switch (self.kind) { + .reference => |p| (Value{ .inner = self.inner, .kernel = self.kernel, .kind = .{ .pointer = p } }).load(0), + .pointer_reference => |p| blk: { + const k = self.kern(); + break :blk k.emit(llvm.load(k.ctx, self.inner, llvm.pointerType(k.ctx, p.space), .{ .alignment = 8 }, k.loc()), .{ .pointer = p }); + }, + else => self, + }; + } + + /// `&x` of a local. + pub fn address(reference: Value) Value { + return switch (reference.kind) { + .reference => |p| .{ .inner = reference.inner, .kernel = reference.kernel, .kind = .{ .pointer = p } }, + else => std.debug.panic("dsl/metal: the address of a {t}", .{reference.kind}), + }; + } + + /// `x = value` of a local, converting `value` to its type. + pub fn assign(self: Value, value: anytype) void { + const k = self.kern(); + switch (self.kind) { + .reference => |p| { + if (p.space != .thread) std.debug.panic("dsl/metal: assigning to a {t} reference", .{p.space}); + (Value{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = p } }).store(0, value); + }, + .pointer_reference => |p| { + const v = k.lift(value); + if (v.kind != .pointer or v.kind.pointer.space != p.space) std.debug.panic("dsl/metal: assigning a {t} to a {t} pointer", .{ v.kind, p.space }); + _ = llvm.store(k.ctx, v.inner, self.inner, .{ .alignment = 8 }, k.loc()).appendTo(k.currentBlock()); + }, + else => std.debug.panic("dsl/metal: assigning to a {t}", .{self.kind}), + } + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("dsl/metal: value has no builder"); + } + + /// The MSL type of a scalar or vector value. + pub fn type_(self: Value) Type { + return switch (self.kind) { + .value => |t| t, + .reference => |p| p.pointee, + else => std.debug.panic("dsl/metal: expected a scalar or vector, got a {t}", .{self.kind}), + }; + } + + pub fn pointer(self: Value) Pointer { + return switch (self.kind) { + .pointer => |p| p, + else => std.debug.panic("dsl/metal: expected a pointer, got a {t}", .{self.kind}), + }; + } + + pub fn add(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), false); + return self.kern().arithmetic(.add, self, rhs); + } + pub fn sub(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), true); + return self.kern().arithmetic(.sub, self, rhs); + } + pub fn mul(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.mul, self, rhs); + } + pub fn div(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.div, self, rhs); + } + pub fn rem(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.rem, self, rhs); + } + pub fn bitAnd(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_and, self, rhs); + } + pub fn bitOr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_or, self, rhs); + } + pub fn bitXor(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_xor, self, rhs); + } + pub fn shl(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.left, self, rhs); + } + pub fn shr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.right, self, rhs); + } + pub fn neg(reference: Value) Value { + const self = reference.rvalue(); + return self.kern().negate(self); + } + /// `~x`. + pub fn bitNot(reference: Value) Value { + const self = reference.rvalue(); + const k = self.kern(); + const v = k.convert(self, self.type_().withScalar(self.type_().scalar.promoted())); + return v.bitXor(k.constant(v.type_(), -1)); + } + + pub fn lt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.lt, self, rhs); + } + pub fn le(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.le, self, rhs); + } + pub fn gt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.gt, self, rhs); + } + pub fn ge(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ge, self, rhs); + } + pub fn eq(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.eq, self, rhs); + } + pub fn ne(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ne, self, rhs); + } + + /// `T(x)`, `(T) x` or `static_cast(x)` on a scalar or vector; on a + /// pointer, `(space T *) p`. + pub fn to(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + return switch (self.kind) { + .pointer => |p| .{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = .{ .space = p.space, .pointee = t } } }, + else => k.convert(self, t), + }; + } + + /// `as_type(x)`: the same bits as another type of the same size. + pub fn asType(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + const from = self.type_(); + if (from.size() != t.size() or from.len == 3 or t.len == 3) + std.debug.panic("dsl/metal: as_type<{s}> of a {s}", .{ t.name(), from.name() }); + if (from.eql(t)) return self; + return k.emit(llvm.bitcast(k.ctx, self.inner, t.mlirType(k.ctx), k.loc()), .{ .value = t }); + } + + /// `v.x`. + pub fn x(reference: Value) Value { + const self = reference.rvalue(); + return self.at(0); + } + pub fn y(reference: Value) Value { + const self = reference.rvalue(); + return self.at(1); + } + pub fn z(reference: Value) Value { + const self = reference.rvalue(); + return self.at(2); + } + pub fn w(reference: Value) Value { + const self = reference.rvalue(); + return self.at(3); + } + + /// `v[i]` of a vector; the index keeps its type, a literal is an `int`. + pub fn at(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + if (!t.isVector()) std.debug.panic("dsl/metal: indexing a {s}", .{t.name()}); + const element = t.element(); + return k.emit(llvm.extractelement(k.ctx, self.inner, k.subscript(i).inner, element.mlirType(k.ctx), k.loc()), .{ .value = element }); + } + + /// The vector with `v[i] = element`, `i` typed as `at` types it; `v.y = + /// element` is `insert(@as(i64, 1), element)`. + pub fn insert(reference: Value, i: anytype, element: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + const e = k.convert(k.lift(element), t.element()); + return k.emit(llvm.insertelement(k.ctx, self.inner, e.inner, k.subscript(i).inner, k.loc()), .{ .value = t }); + } + + /// `p[i]`. + pub fn load(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + const at_ = k.offset(self, i, false); + return k.loadMemory(at_.inner, p.pointee); + } + + /// `p[i] = value`, converting `value` to the pointee type. + pub fn store(reference: Value, i: anytype, value: anytype) void { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + var v = k.convert(k.lift(value), p.pointee); + if (p.pointee.scalar == .bool) v = k.emit(arith.extui(k.ctx, v.inner, p.pointee.memory().mlirType(k.ctx), k.loc()), .{ .value = p.pointee.memory() }); + const at_ = k.offset(self, i, false); + _ = llvm.store(k.ctx, v.inner, at_.inner, .{ .alignment = v.type_().alignment() }, k.loc()).appendTo(k.currentBlock()); + } + + /// `s.name` of a constant struct argument. + pub fn field(reference: Value, comptime name: []const u8) Value { + const self = reference.rvalue(); + const k = self.kern(); + const s = switch (self.kind) { + .constant_struct => |s| s, + else => std.debug.panic("dsl/metal: .{s} of a {t}", .{ name, self.kind }), + }; + const index = s.fieldIndex(name); + const f = s.fields[index]; + if (f.count != 0) std.debug.panic("dsl/metal: struct array field {s}", .{name}); + const gep = llvm.getelementptr(k.ctx, self.inner, &.{ .{ .constant = 0 }, .{ .constant = @intCast(index) } }, k.structType(s), k.loc()); + _ = gep.appendTo(k.currentBlock()); + return k.loadMemory(gep.result(0), f.type); + } + + /// `m.thread_elements()` of a `simdgroup_matrix`: the lane's two + /// elements. + pub fn threadElements(m: Value) Value { + const k = m.kern(); + const t: Type = .vec(Builder.matrixScalar(m), 2); + return k.emit(llvm.shufflevector(k.ctx, m.inner, m.inner, t.mlirType(k.ctx), &.{ 0, 1 }, k.loc()), .{ .value = t }); + } +}; + +/// A loop: `while (c)` and `for (init; c; inc)` branch to the body with +/// `body(c)`, `for (;;)` has its body right away; `next()` ends the body +/// and starts the increment, and `close()` ends the loop. +/// +/// A `break` and the end of the body meet in a latch that reads whether to +/// leave from a local, as clang's cleanup does: Apple's compiler then keeps +/// the loop's test at the top instead of rotating the loop, which is faster +/// (paged attention's decode loop by 2%). +pub const LoopScope = struct { + kernel: *Builder, + cond_block: *mlir.Block, + end_block: *mlir.Block, + loop_annotation: *const llvm.LoopAnnotationAttr, + /// Where a `break` goes after setting `leave`; null until the first one. + latch: ?*mlir.Block = null, + leave: Value = undefined, + exits: bool = false, + + /// Leaves the loop when `cond` is false. + pub fn body(self: *LoopScope, cond: Value) void { + const k = self.kernel; + const body_block = mlir.Block.init(&.{}, &.{}); + k.condBranch(cond, body_block, self.end_block); + self.exits = true; + k.startBlock(body_block); + } + + /// `break;`. + pub fn breakLoop(self: *LoopScope) void { + if (self.kernel.insert == null) return; + self.toLatch(true); + self.exits = true; + } + + fn toLatch(self: *LoopScope, leave: bool) void { + const k = self.kernel; + if (self.latch == null) { + self.latch = mlir.Block.init(&.{}, &.{}); + self.leave = k.local(.bool, null); + } + self.leave.assign(leave); + _ = k.branch(self.latch.?); + } + + /// Ends the body: the increment of `for (...; ...; inc)` follows. + pub fn next(self: *LoopScope) void { + const k = self.kernel; + const inc = mlir.Block.init(&.{}, &.{}); + if (self.latch) |latch| { + if (k.insert != null) self.toLatch(false); + k.startBlock(latch); + k.condBranch(self.leave, self.end_block, inc); + self.latch = null; + } else _ = k.branch(inc); + k.startBlock(inc); + } + + pub fn close(self: *LoopScope) void { + const k = self.kernel; + if (self.latch != null) self.next(); + k.branchLoop(self.cond_block, self.loop_annotation); + if (self.exits) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// `for (T i = lower; i < upper; i += step)`: `i` is the local `iv`, the +/// condition and the increment follow C++'s conversions, and a step of 1 is +/// `++i`. +pub const ForScope = struct { + loop: LoopScope, + iv: Value, + step: Step, + + const Step = union(enum) { one, value: Value, int: i64 }; + + pub fn breakLoop(self: *ForScope) void { + self.loop.breakLoop(); + } + + pub fn close(self: *ForScope) void { + const k = self.loop.kernel; + self.loop.next(); + if (k.insert != null) switch (self.step) { + .one => self.iv.assign(k.increment(self.iv)), + .value => |v| { + const s = v.rvalue(); + self.iv.assign(self.iv.add(s)); + }, + .int => |n| self.iv.assign(self.iv.add(k.constant(if (std.math.cast(i32, n) != null) .int else .long, n))), + }; + self.loop.close(); + } +}; + +/// `if (c) { ... }`, or `if (c) { ... } else { ... }` with `elseBranch()` +/// between the two. +pub const IfScope = struct { + kernel: *Builder, + else_block: ?*mlir.Block, + end_block: *mlir.Block, + joins: bool = false, + + pub fn elseBranch(self: *IfScope) void { + const k = self.kernel; + const else_block = self.else_block orelse @panic("dsl/metal: elseBranch of an if without else (open it with openIfElse(c, .{})), or a second one"); + if (k.branch(self.end_block)) self.joins = true; + k.startBlock(else_block); + self.else_block = null; + } + + pub fn close(self: *IfScope) void { + const k = self.kernel; + if (k.branch(self.end_block)) self.joins = true; + if (self.joins) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// A condition of `&&`, `||` and `!` terms, short-circuited: a term's code +/// runs only when the terms before it leave the outcome open. `and_(x)` is +/// `x && ...`, `or_(x)` is `x || ...`; the last term opens what the condition +/// guards. A group of `&&` terms ends a `||` chain and the +/// other way around, as in `a || (b && c)`. +pub const Condition = struct { + kernel: *Builder, + true_block: *mlir.Block, + false_block: *mlir.Block, + + pub fn and_(self: *Condition, x: Value) void { + self.term(x, false, .@"and"); + } + pub fn andNot(self: *Condition, x: Value) void { + self.term(x, true, .@"and"); + } + pub fn or_(self: *Condition, x: Value) void { + self.term(x, false, .@"or"); + } + pub fn orNot(self: *Condition, x: Value) void { + self.term(x, true, .@"or"); + } + + fn term(self: *Condition, x: Value, negate: bool, comptime op: enum { @"and", @"or" }) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const on_true, const on_false = switch (op) { + .@"and" => .{ next, self.false_block }, + .@"or" => .{ self.true_block, next }, + }; + if (negate) k.condBranch(x, on_false, on_true) else k.condBranch(x, on_true, on_false); + k.startBlock(next); + } + + fn last(self: *Condition, x: Value, negate: bool) void { + if (negate) self.kernel.condBranch(x, self.false_block, self.true_block) else self.kernel.condBranch(x, self.true_block, self.false_block); + } + + /// `if (... x) { ... }`. + pub fn openIf(self: *Condition, x: Value) IfScope { + return self.opened(x, false, false); + } + /// `if (... !x) { ... }`. + pub fn openIfNot(self: *Condition, x: Value) IfScope { + return self.opened(x, true, false); + } + /// `if (... x) { ... } else { ... }`; a chain has no result types. + pub fn openIfElse(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, false, true); + } + /// `if (... !x) { ... } else { ... }`. + pub fn openIfElseNot(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, true, true); + } + + fn noResults(result_types: anytype) void { + if (common.arity(@TypeOf(result_types), "Condition.openIfElse: result_types") != 0) + @compileError("dsl/metal: a condition chain opens no `c ? a : b`"); + } + + fn opened(self: *Condition, x: Value, negate: bool, with_else: bool) IfScope { + const k = self.kernel; + self.last(x, negate); + k.startBlock(self.true_block); + if (!with_else) return .{ .kernel = k, .else_block = null, .end_block = self.false_block, .joins = true }; + return .{ .kernel = k, .else_block = self.false_block, .end_block = mlir.Block.init(&.{}, &.{}) }; + } +}; + +/// `a && b && ...` or `a || b || ...` as a bool, short-circuited: a +/// term's code runs only when the terms before it leave the result open. +/// `term(x)` adds a term followed by more, `close(x)` the last one. +pub const Logical = struct { + kernel: *Builder, + op: enum { @"and", @"or" }, + end_block: *mlir.Block, + + pub fn term(self: *Logical, x: Value) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const c = k.convert(x, .bool); + const decided: []const *const mlir.Value = &.{k.constant(.bool, self.op == .@"or").inner}; + _ = switch (self.op) { + .@"and" => cf.cond_br(k.ctx, c.inner, next, &.{}, self.end_block, decided, .{}, k.loc()), + .@"or" => cf.cond_br(k.ctx, c.inner, self.end_block, decided, next, &.{}, .{}, k.loc()), + }.appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(next); + } + + pub fn close(self: *Logical, x: Value) Value { + const k = self.kernel; + const c = k.convert(x, .bool); + _ = cf.br(k.ctx, self.end_block, &.{c.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(self.end_block); + return .{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = .bool } }; + } +}; + +/// `c ? a : b` evaluating one operand: the true operand's code, then +/// `yieldThen(.{a})`, the false operand's code, then `yieldElse(.{b})`; +/// `results[0]` is the value. +pub const TernaryScope = struct { + kernel: *Builder, + type: Type, + else_block: *mlir.Block, + end_block: *mlir.Block, + results: [1]Value = undefined, + + pub fn yieldThen(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldThen"); + k.startBlock(self.else_block); + } + + pub fn yieldElse(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldElse"); + k.startBlock(self.end_block); + self.results = .{.{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = self.type } }}; + } + + fn branchEnd(self: *TernaryScope, values: anytype, comptime what: []const u8) void { + if (common.arity(@TypeOf(values), "TernaryScope." ++ what) != 1) @compileError("TernaryScope." ++ what ++ " takes one value"); + const k = self.kernel; + const v = k.convert(k.lift(values[0]), self.type); + _ = cf.br(k.ctx, self.end_block, &.{v.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + } +}; + +/// What `Builder.openIfElse` opens for `n` result types. +fn IfElseScope(comptime n: usize) type { + return switch (n) { + 0 => IfScope, + 1 => TernaryScope, + else => @compileError("dsl/metal: openIfElse has one result at most, as `c ? a : b`"), + }; +} + +/// A loop pragma: `full` is `#pragma clang loop unroll(full)`, `enable` +/// `#pragma unroll`. +pub const Unroll = enum { full, enable }; + +pub const LoopOpts = struct { + unroll: ?Unroll = null, +}; + +/// MSL's `metal::fast` and `metal::precise` functions. +pub const MathMode = enum { default, fast, precise }; + +/// `metal::fast::` or `metal::precise::`. +pub const Namespace = struct { + b: *Builder, + mode: MathMode, + + pub fn exp(self: Namespace, x: Value) Value { + return self.b.floatMath("exp", self.mode, .{x}); + } + pub fn exp2(self: Namespace, x: Value) Value { + return self.b.floatMath("exp2", self.mode, .{x}); + } + pub fn log(self: Namespace, x: Value) Value { + return self.b.floatMath("log", self.mode, .{x}); + } + pub fn log2(self: Namespace, x: Value) Value { + return self.b.floatMath("log2", self.mode, .{x}); + } + pub fn tanh(self: Namespace, x: Value) Value { + return self.b.floatMath("tanh", self.mode, .{x}); + } + pub fn sqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("sqrt", self.mode, .{x}); + } + pub fn rsqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("rsqrt", self.mode, .{x}); + } + pub fn sin(self: Namespace, x: Value) Value { + return self.b.floatMath("sin", self.mode, .{x}); + } + pub fn cos(self: Namespace, x: Value) Value { + return self.b.floatMath("cos", self.mode, .{x}); + } + pub fn max(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmax", self.mode, .{ a, b }); + } + pub fn min(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmin", self.mode, .{ a, b }); + } +}; + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + fast_math: bool, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + /// The kernel's body; blocks join it in the order they start. + body: ?*mlir.Region = null, + /// The block code goes to; null after a `break` or a `return`. + insert: ?*mlir.Block = null, + arguments: std.ArrayList(Argument) = .empty, + + pub const Opts = struct { + /// MSL's default; `-fno-fast-math` otherwise. + fast_math: bool = true, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8, opts: Opts) Builder { + const module: *mlir.Module = .init(.unknown(ctx)); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + .fast_math = opts.fast_math, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Declares the kernel's arguments, in order, from a struct literal of + /// `ArgSpec`s, and returns them by name. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!common.NamedArgs(@TypeOf(spec), Value) { + const ctx = self.ctx; + const location = self.loc(); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const arena = self.arena.allocator(); + var arg_types: [fields.len]*const mlir.Type = undefined; + var arg_attrs: [fields.len]*const mlir.Attribute = undefined; + var kinds: [fields.len]Value.Kind = undefined; + var buffers: u32 = 0; + var threadgroups: u32 = 0; + inline for (fields, 0..) |name, i| { + const arg = argSpec(@field(spec, name)); + const opts: air.ArgumentOpts = .{ .name = name }; + switch (arg) { + .buffer => |buffer| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.device); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .device, + .access = if (buffer.const_) .read else .read_write, + .elementType = buffer.type.memory().mlirType(ctx), + .typeName = buffer.type.name(), + .typeSize = buffer.type.size(), + .typeAlignment = buffer.type.alignment(), + .noalias_ = buffer.restrict, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .device, .pointee = buffer.type } }; + try self.arguments.append(arena, .{ .name = name, .kind = .buffer, .index = buffers }); + buffers += 1; + }, + .constant => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + .bufferSize = t.size(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .reference = .{ .space = .constant, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = t.size() }); + buffers += 1; + }, + .constant_struct => |s| { + var struct_fields = try arena.alloc(*const air.StructFieldAttr, s.fields.len); + for (s.fields, 0..) |f, j| struct_fields[j] = air.StructFieldAttr.get(ctx, .{ + .offset = s.offset(j), + .size = f.type.size(), + .count = f.count, + .typeName = f.type.name(), + .name = f.name, + }) catch return error.InvalidMlir; + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = self.structType(s), + .typeName = s.name, + .typeSize = s.size(), + .typeAlignment = s.alignment(), + .bufferSize = s.size(), + .fields = struct_fields, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .constant_struct = s }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = s.size() }); + buffers += 1; + }, + .threadgroup => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.threadgroup); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = threadgroups, + .addressSpace = .threadgroup, + .access = .read_write, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .threadgroup, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .threadgroup, .index = threadgroups }); + threadgroups += 1; + }, + inline else => |t, builtin| { + arg_types[i] = t.mlirType(ctx); + arg_attrs[i] = air.builtinArgumentAttr(ctx, @field(air.Builtin, @tagName(builtin)), opts) catch return error.InvalidMlir; + kinds[i] = .{ .value = t }; + }, + } + } + var locs: [fields.len]*const mlir.Location = @splat(location); + const block = mlir.Block.init(&arg_types, &locs); + self.entry_block = block; + self.func_op = air.kernel(ctx, .{ + .name = self.name, + .block = block, + .args_attributes = &arg_attrs, + .location = location, + .verify = false, + }).appendTo(self.module.body()); + self.body = block.parentRegion(); + self.insert = block; + + var out: common.NamedArgs(@TypeOf(spec), Value) = undefined; + inline for (fields, 0..) |name, i| @field(out, name) = .{ .inner = block.argument(i), .kernel = self, .kind = kinds[i] }; + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// The block code goes to: a fresh, unreachable one after a `break` or + /// a `return`. + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.insert == null) self.startBlock(mlir.Block.init(&.{}, &.{})); + return self.insert.?; + } + + fn startBlock(self: *Builder, block: *mlir.Block) void { + self.body.?.appendOwnedBlock(block); + self.insert = block; + } + + /// Branches to `dest` unless the code here is unreachable; true if it did. + fn branch(self: *Builder, dest: *mlir.Block) bool { + const block = self.insert orelse return false; + _ = cf.br(self.ctx, dest, &.{}, self.loc()).appendTo(block); + self.insert = null; + return true; + } + + fn condBranch(self: *Builder, cond: Value, on_true: *mlir.Block, on_false: *mlir.Block) void { + const c = self.convert(cond, .bool); + _ = cf.cond_br(self.ctx, c.inner, on_true, &.{}, on_false, &.{}, .{}, self.loc()).appendTo(self.currentBlock()); + self.insert = null; + } + + /// The back edge of a loop, which carries its `llvm.loop` metadata. + fn branchLoop(self: *Builder, dest: *mlir.Block, annotation: *const llvm.LoopAnnotationAttr) void { + const block = self.insert orelse return; + _ = llvm.br(self.ctx, dest, &.{}, .{ .loop_annotation = annotation }, self.loc()).appendTo(block); + self.insert = null; + } + + /// A thread-memory slot of `t`, at the top of the entry block. + fn allocate(self: *Builder, t: *const mlir.Type, alignment: u64) *const mlir.Value { + const ctx = self.ctx; + const entry = self.entry_block.?; + const size = arith.constant_int(ctx, 1, .int(ctx, .i32), self.loc()); + const alloca = llvm.alloca(ctx, size.result(0), llvm.pointerType(ctx, AddressSpace.thread), t, .{ .alignment = alignment }, self.loc()); + if (entry.firstOperation()) |first| entry.insertOwnedOperationBefore(first, alloca) else _ = alloca.appendTo(entry); + entry.insertOwnedOperationBefore(alloca, size); + return alloca.result(0); + } + + /// `T x;`, or `T x = init;`: a local, read where it is used and written by + /// `assign`. + pub fn local(self: *Builder, t: Type, init: anytype) Value { + const memory = t.memory(); + const x: Value = .{ .inner = self.allocate(memory.mlirType(self.ctx), memory.alignment()), .kernel = self, .kind = .{ .reference = .{ .space = .thread, .pointee = t } } }; + if (@TypeOf(init) != @TypeOf(null)) x.assign(init); + return x; + } + + /// `T *p = init;`: a pointer local. + pub fn localPointer(self: *Builder, init: Value) Value { + const p: Pointer = switch (init.kind) { + .pointer, .pointer_reference => |q| q, + else => std.debug.panic("dsl/metal: a pointer local of a {t}", .{init.kind}), + }; + const x: Value = .{ .inner = self.allocate(llvm.pointerType(self.ctx, p.space), 8), .kernel = self, .kind = .{ .pointer_reference = p } }; + x.assign(init); + return x; + } + + /// A `simdgroup_matrix` whose lane holds `elements`, two values + /// converted to `T`: what assigning `m.thread_elements()` makes. + pub fn matrix(self: *Builder, scalar: Scalar, elements: anytype) Value { + const v = self.convert(self.lift(elements), .vec(scalar, 2)); + var mask: [64]i32 = @splat(-1); + mask[0..2].* = .{ 0, 1 }; + return self.emit(llvm.shufflevector(self.ctx, v.inner, v.inner, self.matrixType(scalar), &mask, self.loc()), .{ .matrix = scalar }); + } + + /// `++i`: a narrow integer steps in its own type, as C++ steps it. + fn increment(self: *Builder, i: Value) Value { + const v = i.rvalue(); + const t = v.type_(); + if (t.scalar.isFloat() or t.scalar.bits() >= 32) return v.add(1); + return self.emit(arith.addi(self.ctx, v.inner, self.constant(t, 1).inner, self.loc()), .{ .value = t }); + } + + fn emit(self: *Builder, op: *mlir.Operation, kind: Value.Kind) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self, .kind = kind }; + } + + fn fastmath(self: *const Builder) air.FloatOpts { + return .{ .fastmath = if (self.fast_math) .fast else .none }; + } + + fn setFastmath(self: *Builder, op: *mlir.Operation) void { + if (!self.fast_math) return; + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); + } + + fn structType(self: *Builder, s: Struct) *const mlir.Type { + var fields: [32]*const mlir.Type = undefined; + for (s.fields, 0..) |f, i| { + const t = f.type.memory().mlirType(self.ctx); + fields[i] = if (f.count != 0) llvm.arrayType(t, f.count) else t; + } + var name: std.ArrayList(u8) = .empty; + name.print(self.arena.allocator(), "struct.{s}", .{s.name}) catch @panic("dsl/metal: out of memory"); + return llvm.identifiedStructType(self.ctx, name.items, fields[0..s.fields.len]) orelse + std.debug.panic("dsl/metal: two structs named {s}", .{s.name}); + } + + /// The MSL type of a Zig number, as `lift` gives it. + fn literalType(value: anytype) Type { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int => if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .int else .long, + .comptime_float => .float, + .int => |info| switch (info.bits) { + 8 => if (info.signedness == .signed) .char else .uchar, + 16 => if (info.signedness == .signed) .short else .ushort, + 32 => if (info.signedness == .signed) .int else .uint, + 64 => if (info.signedness == .signed) .long else .ulong, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(@TypeOf(value))), + }, + else => @compileError("dsl/metal: expected a literal, got " ++ @typeName(@TypeOf(value))), + }; + } + + /// A Zig value as an MSL literal: integers are `int` (`long` when they do + /// not fit), floats are `float`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value.rvalue(); + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float, .int => self.constant(literalType(value), value), + .bool => self.constant(.bool, value), + .float => |info| self.constant(switch (info.bits) { + 16 => Type.half, + 32 => Type.float, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(T)), + }, value), + else => @compileError("dsl/metal: expected a Value or a number, got " ++ @typeName(T)), + }; + } + + /// A constant of type `t`, splat across a vector. + pub fn constant(self: *Builder, t: Type, value: anytype) Value { + const ctx = self.ctx; + const ty = t.mlirType(ctx); + const op = if (t.scalar.isFloat()) blk: { + const v: f64 = switch (@typeInfo(@TypeOf(value))) { + .int, .comptime_int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), + else => @floatCast(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_f(ctx, v, ty, self.loc()); + break :blk switch (t.scalar) { + .half => arith.constant_float(ctx, v, .f16, self.loc()), + .bfloat => arith.constant_float(ctx, v, .bf16, self.loc()), + else => arith.constant_float(ctx, v, .f32, self.loc()), + }; + } else blk: { + const v: i64 = switch (@typeInfo(@TypeOf(value))) { + .bool => @intFromBool(value), + .int, .comptime_int => if (@TypeOf(value) == u64) @bitCast(value) else value, + else => @intFromFloat(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_i(ctx, v, ty, self.loc()); + break :blk arith.constant_int(ctx, v, ty, self.loc()); + }; + return self.emit(op, .{ .value = t }); + } + + /// `T(a, b, ...)` from scalars, or `T(x)` splat from one scalar. + pub fn vec(self: *Builder, t: Type, elements: anytype) Value { + const fields = @typeInfo(@TypeOf(elements)).@"struct".field_names; + if (fields.len == 1) return self.convert(self.lift(@field(elements, fields[0])), t); + if (fields.len != t.len) std.debug.panic("dsl/metal: {d} elements of a {s}", .{ fields.len, t.name() }); + var inner: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |name, i| inner[i] = self.convert(self.lift(@field(elements, name)), t.element()).inner; + return self.emit(vector.from_elements(self.ctx, &inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + fn splat(self: *Builder, scalar: Value, t: Type) Value { + return self.emit(vector.broadcast(self.ctx, scalar.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// The conversion of `v` to `to`, as C++ converts: integer widths extend + /// by the source's signedness or truncate, integers and floats convert + /// through `air.convert`, and a scalar converts to a vector by splatting. + pub fn convert(self: *Builder, reference: Value, to: Type) Value { + const v = reference.rvalue(); + const ctx = self.ctx; + const from = v.type_(); + if (from.eql(to)) return v; + if (from.len != to.len) { + if (!from.isVector()) return self.splat(self.convert(v, to.element()), to); + std.debug.panic("dsl/metal: cannot convert a {s} to a {s}", .{ from.name(), to.name() }); + } + const fs = from.scalar; + const ts = to.scalar; + const ty = to.mlirType(ctx); + const kind: Value.Kind = .{ .value = to }; + if (ts == .bool) return v.ne(self.constant(from, 0)); + if (fs == .bool) { + if (ts.isFloat()) return self.convert(self.convert(v, to.withScalar(.int)), to); + return self.emit(arith.extui(ctx, v.inner, ty, self.loc()), kind); + } + if (fs.isFloat() and ts.isFloat()) { + if (!from.isVector() and (fs == .float or ts == .float)) { + const op = if (ts == .float) arith.extf(ctx, v.inner, ty, self.loc()) else arith.truncf(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + return self.emit(air.convert(ctx, v.inner, ty, self.fastmath(), self.loc()), kind); + } + if (fs.isFloat()) { + const op = if (ts.isUnsigned()) air.convert_u(ctx, v.inner, ty, .{}, self.loc()) else air.convert_s(ctx, v.inner, ty, .{}, self.loc()); + return self.emit(op, kind); + } + if (ts.isFloat()) { + const op = if (fs.isUnsigned()) air.convert_u(ctx, v.inner, ty, self.fastmath(), self.loc()) else air.convert_s(ctx, v.inner, ty, self.fastmath(), self.loc()); + return self.emit(op, kind); + } + if (fs.bits() == ts.bits()) return .{ .inner = v.inner, .kernel = self, .kind = kind }; + if (fs.bits() > ts.bits()) return self.emit(arith.trunci(ctx, v.inner, ty, self.loc()), kind); + const op = if (fs.isUnsigned()) arith.extui(ctx, v.inner, ty, self.loc()) else arith.extsi(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + + /// The operands of a binary operator after the usual arithmetic + /// conversions; a scalar operand of a vector takes its element type. + fn usualConversions(self: *Builder, a: Value, b_: anytype) [2]Value { + const b = self.lift(b_); + const ta = a.type_(); + const tb = b.type_(); + const converted: Type = if (ta.isVector() and tb.isVector()) blk: { + if (!ta.eql(tb)) std.debug.panic("dsl/metal: {s} and {s} operands", .{ ta.name(), tb.name() }); + break :blk ta; + } else if (ta.isVector()) ta else if (tb.isVector()) tb else .{ .scalar = types.commonScalar(ta.scalar, tb.scalar) }; + return .{ self.convert(a, converted), self.convert(b, converted) }; + } + + const ArithmeticOp = enum { add, sub, mul, div, rem, bit_and, bit_or, bit_xor }; + + fn arithmetic(self: *Builder, comptime op: ArithmeticOp, a: Value, b: anytype) Value { + const ctx = self.ctx; + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const loc_ = self.loc(); + if (t.scalar.isFloat()) { + const result = switch (op) { + .add => arith.addf(ctx, l.inner, r.inner, loc_), + .sub => arith.subf(ctx, l.inner, r.inner, loc_), + .mul => arith.mulf(ctx, l.inner, r.inner, loc_), + .div => arith.divf(ctx, l.inner, r.inner, loc_), + else => std.debug.panic("dsl/metal: {t} of floats", .{op}), + }; + self.setFastmath(result); + return self.emit(result, .{ .value = t }); + } + const unsigned = t.scalar.isUnsigned(); + const result = switch (op) { + .add => arith.addi(ctx, l.inner, r.inner, loc_), + .sub => arith.subi(ctx, l.inner, r.inner, loc_), + .mul => arith.muli(ctx, l.inner, r.inner, loc_), + .div => if (unsigned) arith.divui(ctx, l.inner, r.inner, loc_) else arith.divsi(ctx, l.inner, r.inner, loc_), + .rem => if (unsigned) arith.remui(ctx, l.inner, r.inner, loc_) else arith.remsi(ctx, l.inner, r.inner, loc_), + .bit_and => arith.andi(ctx, l.inner, r.inner, loc_), + .bit_or => arith.ori(ctx, l.inner, r.inner, loc_), + .bit_xor => arith.xori(ctx, l.inner, r.inner, loc_), + }; + // Signed overflow is undefined in C++. + if (!unsigned and (op == .add or op == .sub or op == .mul)) { + result.setAttribute(arith.overflowFlags(ctx, .{ .nsw = true }) catch unreachable); + } + return self.emit(result, .{ .value = t }); + } + + /// `a << b` or `a >> b`. MSL shifts by the amount modulo the width: + /// clang casts the promoted amount to `a`'s promoted type as unsigned, + /// then masks it, which folds away for a number within the width. + fn shift(self: *Builder, comptime direction: enum { left, right }, a: Value, b: anytype) Value { + const t = a.type_().withScalar(a.type_().scalar.promoted()); + const l = self.convert(a, t); + const mask = t.scalar.bits() - 1; + const r = if (@TypeOf(b) == Value) blk: { + const rb = b.rvalue(); + const amount = self.shiftAmount(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + break :blk self.emit(arith.andi(self.ctx, amount.inner, self.constant(t, mask).inner, self.loc()), .{ .value = t }); + } else blk: { + if (b < 0 or b > mask) std.debug.panic("dsl/metal: a shift of a {s} by {d}, past its width", .{ t.name(), b }); + const rb = self.lift(b); + break :blk self.convert(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + }; + const loc_ = self.loc(); + const op = switch (direction) { + .left => arith.shli(self.ctx, l.inner, r.inner, loc_), + .right => if (t.scalar.isUnsigned()) arith.shrui(self.ctx, l.inner, r.inner, loc_) else arith.shrsi(self.ctx, l.inner, r.inner, loc_), + }; + return self.emit(op, .{ .value = t }); + } + + /// A promoted shift amount `v` as the shifted type `t`: a scalar of a + /// vector converts and splats, as C++ converts it; otherwise the cast is + /// unsigned, as clang casts it. + fn shiftAmount(self: *Builder, v: Value, t: Type) Value { + const from = v.type_(); + if (from.len != t.len) return self.convert(v, t); + if (from.scalar.bits() == t.scalar.bits()) return .{ .inner = v.inner, .kernel = self, .kind = .{ .value = t } }; + const ty = t.mlirType(self.ctx); + const op = if (from.scalar.bits() > t.scalar.bits()) arith.trunci(self.ctx, v.inner, ty, self.loc()) else arith.extui(self.ctx, v.inner, ty, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn negate(self: *Builder, a: Value) Value { + const t = a.type_(); + if (t.scalar.isFloat()) { + const op = arith.negf(self.ctx, a.inner, self.loc()); + self.setFastmath(op); + return self.emit(op, .{ .value = t }); + } + const x = self.convert(a, t.withScalar(t.scalar.promoted())); + return self.constant(x.type_(), 0).sub(x); + } + + const Comparison = enum { lt, le, gt, ge, eq, ne }; + + fn compare(self: *Builder, comptime op: Comparison, a: Value, b: anytype) Value { + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const result_type = t.withScalar(.bool); + if (t.scalar.isFloat()) { + const predicate: arith.CmpFPredicate = switch (op) { + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + .eq => .oeq, + .ne => .une, + }; + const result = arith.cmpf(self.ctx, predicate, l.inner, r.inner, self.loc()); + self.setFastmath(result); + return self.emit(result, .{ .value = result_type }); + } + const unsigned = t.scalar.isUnsigned(); + const predicate: arith.CmpIPredicate = switch (op) { + .lt => if (unsigned) .ult else .slt, + .le => if (unsigned) .ule else .sle, + .gt => if (unsigned) .ugt else .sgt, + .ge => if (unsigned) .uge else .sge, + .eq => .eq, + .ne => .ne, + }; + return self.emit(arith.cmpi(self.ctx, predicate, l.inner, r.inner, self.loc()), .{ .value = result_type }); + } + + /// `i` of `p + i` and `p - i`, promoted as C++ promotes an integer + /// operand; `p[i]` and a vector subscript take `i` as it is. + fn promoted(self: *Builder, i: anytype) Value { + const v = self.subscript(i); + const t = v.type_(); + return self.convert(v, t.withScalar(t.scalar.promoted())); + } + + /// A vector subscript, of its own type as clang keeps it: a literal is an + /// `int`, a `ushort` an `i16`, a `bool` an `i1`. + fn subscript(self: *Builder, i: anytype) Value { + const v = self.lift(i); + const t = v.type_(); + if (t.isVector() or t.scalar.isFloat()) std.debug.panic("dsl/metal: an index of a {s}", .{t.name()}); + return v; + } + + /// A `t` read from `p`, where it is a `t.memory()`. + fn loadMemory(self: *Builder, p: *const mlir.Value, t: Type) Value { + const memory = t.memory(); + const x = self.emit(llvm.load(self.ctx, p, memory.mlirType(self.ctx), .{ .alignment = memory.alignment() }, self.loc()), .{ .value = memory }); + if (t.scalar != .bool) return x; + return self.emit(arith.trunci(self.ctx, x.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// `p + i` (`p - i` when `negative`) in elements of the pointee type. + fn offset(self: *Builder, p: Value, i: anytype, negative: bool) Value { + if (@TypeOf(i) == comptime_int and i == 0) return p; + const pointee = switch (p.kind) { + .pointer => |ptr| ptr.pointee, + else => std.debug.panic("dsl/metal: pointer arithmetic on a {t}", .{p.kind}), + }; + var index = self.lift(i); + if (index.type_().scalar.isFloat() or index.type_().isVector()) std.debug.panic("dsl/metal: pointer offset by a {s}", .{index.type_().name()}); + index = self.convert(index, if (index.type_().scalar.isUnsigned()) .ulong else .long); + if (negative) index = self.constant(.long, 0).sub(index); + const gep = llvm.getelementptr(self.ctx, p.inner, &.{.{ .value = index.inner }}, pointee.memory().mlirType(self.ctx), self.loc()); + return self.emit(gep, p.kind); + } + + /// `cond ? a : b` of evaluated operands, a select; + /// `openIfElse(cond, .{T})` evaluates one operand. + pub fn ternary(self: *Builder, cond: Value, a: anytype, b: anytype) Value { + const l, const r = self.usualConversions(self.lift(a), b); + const c = self.convert(cond, if (l.type_().isVector() and cond.type_().isVector()) l.type_().withScalar(.bool) else .bool); + return self.emit(arith.select(self.ctx, c.inner, l.inner, r.inner, self.loc()), l.kind); + } + + fn openTernary(self: *Builder, cond: Value, t: Type) TernaryScope { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + var loc_ = [1]*const mlir.Location{self.loc()}; + const end_block = mlir.Block.init(&.{t.mlirType(self.ctx)}, &loc_); + self.condBranch(cond, then_block, else_block); + self.startBlock(then_block); + return .{ .kernel = self, .type = t, .else_block = else_block, .end_block = end_block }; + } + + /// `T a[n] = {};`, zeroed as clang zeroes it. + pub fn zeroedArray(self: *Builder, t: Type, n: u32) Value { + const a = self.array(t, n); + const len = self.constant(.long, @as(i64, n) * @as(i64, @intCast(t.size()))); + const operands = [_]*const mlir.Value{ a.inner, self.constant(.char, 0).inner, len.inner, self.constant(.bool, false).inner }; + _ = llvm.call_intrinsic(self.ctx, "llvm.memset.p0.i64", &operands, .{}, self.loc()).appendTo(self.currentBlock()); + return a; + } + + /// `T a[n];` in thread memory, as a pointer to its first element. + pub fn array(self: *Builder, t: Type, n: u32) Value { + return .{ .inner = self.allocate(llvm.arrayType(t.memory().mlirType(self.ctx), n), t.alignment()), .kernel = self, .kind = .{ .pointer = .{ .space = .thread, .pointee = t } } }; + } + + fn floatMath(self: *Builder, comptime name: []const u8, mode: MathMode, args: anytype) Value { + const x = self.lift(args[0]); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) + std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }); + const use_fast = switch (mode) { + .default => self.fast_math and t.scalar == .float, + .fast => true, + .precise => false, + }; + if (use_fast and t.scalar != .float) std.debug.panic("dsl/metal: fast::{s} of a {s}", .{ name, t.name() }); + const opts = self.fastmath(); + const loc_ = self.loc(); + const op = switch (args.len) { + 1 => if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, opts, loc_), + 2 => blk: { + const y = self.convert(self.lift(args[1]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, opts, loc_); + }, + 3 => blk: { + const y = self.convert(self.lift(args[1]), t); + const z = self.convert(self.lift(args[2]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_); + }, + else => unreachable, + }; + return self.emit(op, .{ .value = t }); + } + + /// `metal::fast::f`: `b.fast().exp2(x)`. + pub fn fast(self: *Builder) Namespace { + return .{ .b = self, .mode = .fast }; + } + + /// `metal::precise::f`: `b.precise().tanh(x)`. + pub fn precise(self: *Builder) Namespace { + return .{ .b = self, .mode = .precise }; + } + + pub fn exp(self: *Builder, x: Value) Value { + return self.floatMath("exp", .default, .{x}); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.floatMath("exp2", .default, .{x}); + } + pub fn log(self: *Builder, x: Value) Value { + return self.floatMath("log", .default, .{x}); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.floatMath("log2", .default, .{x}); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.floatMath("tanh", .default, .{x}); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.floatMath("sqrt", .default, .{x}); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.floatMath("rsqrt", .default, .{x}); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.floatMath("ceil", .default, .{x}); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.floatMath("floor", .default, .{x}); + } + pub fn round(self: *Builder, x: Value) Value { + return self.floatMath("round", .default, .{x}); + } + pub fn trunc(self: *Builder, x: Value) Value { + return self.floatMath("trunc", .default, .{x}); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.floatMath("sin", .default, .{x}); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.floatMath("cos", .default, .{x}); + } + pub fn pow(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("pow", .default, .{ x, y }); + } + pub fn fmod(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("fmod", .default, .{ x, y }); + } + + /// `fma(a, b, c)` of floats. + pub fn fma(self: *Builder, a_: Value, b: anytype, c: anytype) Value { + const a = a_.rvalue(); + const t = a.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: fma of a {s}", .{t.name()}); + const y = self.convert(self.lift(b), t); + const z = self.convert(self.lift(c), t); + return self.emit(air.fma(self.ctx, a.inner, y.inner, z.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + + /// `dot(a, b)` of float vectors. + pub fn dot(self: *Builder, a_: Value, b_: Value) Value { + const a = a_.rvalue(); + const b = b_.rvalue(); + const t = a.type_(); + if (!t.eql(b.type_()) or !t.isVector()) std.debug.panic("dsl/metal: dot of {s} and {s}", .{ t.name(), b.type_().name() }); + return self.emit(air.dot(self.ctx, a.inner, b.inner, self.fastmath(), self.loc()), .{ .value = t.element() }); + } + + /// `abs(x)`. + pub fn abs(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (t.scalar.isFloat()) return self.floatMath("fabs", .default, .{x}); + if (t.scalar.isUnsigned()) return x; + return self.emit(air.abs_s(self.ctx, x.inner, self.loc()), .{ .value = t }); + } + + /// The operands of an MSL function taking several values of one type, + /// read, with literals converted to the other operand's type. + fn sameType(self: *Builder, a: anytype, b: anytype) [2]Value { + if (@TypeOf(a) != Value) { + const r = self.lift(b); + return .{ self.convert(self.lift(a), r.type_()), r }; + } + const l = a.rvalue(); + const r = if (@TypeOf(b) == Value) b.rvalue() else self.convert(self.lift(b), l.type_()); + if (!l.type_().eql(r.type_())) std.debug.panic("dsl/metal: {s} and {s} arguments", .{ l.type_().name(), r.type_().name() }); + return .{ l, r }; + } + + /// `max(a, b)`. + pub fn max(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmax", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.max_u(self.ctx, l.inner, r.inner, self.loc()) else air.max_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `min(a, b)`. + pub fn min(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmin", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.min_u(self.ctx, l.inner, r.inner, self.loc()) else air.min_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `clamp(x, lo, hi)`. + pub fn clamp(self: *Builder, x_: Value, lo: anytype, hi: anytype) Value { + const x = x_.rvalue(); + const t = x.type_(); + const l = self.convert(self.lift(lo), t); + const h = self.convert(self.lift(hi), t); + if (t.scalar.isFloat()) return self.floatMath("clamp", .default, .{ x, l, h }); + const op = if (t.scalar.isUnsigned()) air.clamp_u(self.ctx, x.inner, l.inner, h.inner, self.loc()) else air.clamp_s(self.ctx, x.inner, l.inner, h.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn simdType(x: Value, comptime name: []const u8) Type { + const t = x.type_(); + switch (t.scalar) { + .half, .float, .char, .uchar, .short, .ushort, .int, .uint => {}, + else => std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }), + } + return t; + } + + fn simdReduce(self: *Builder, comptime name: []const u8, x_: Value) Value { + const x = x_.rvalue(); + const t = simdType(x, name); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// A function moving `x` across lanes; `x` is frozen, so that a poison + /// lane does not poison the others. + fn shuffle(self: *Builder, comptime name: []const u8, x_: Value, lane: anytype) Value { + const read = x_.rvalue(); + const t = simdType(read, name); + const x = self.emit(llvm.freeze(self.ctx, read.inner, self.loc()), .{ .value = t }); + const l = self.convert(self.lift(lane), .ushort); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, l.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, l.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, l.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + pub fn simdSum(self: *Builder, x: Value) Value { + return self.simdReduce("simd_sum", x); + } + pub fn simdMax(self: *Builder, x: Value) Value { + return self.simdReduce("simd_max", x); + } + pub fn simdMin(self: *Builder, x: Value) Value { + return self.simdReduce("simd_min", x); + } + pub fn simdPrefixExclusiveSum(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: simd_prefix_exclusive_sum of a {s}", .{t.name()}); + return self.emit(air.simd_prefix_exclusive_sum(self.ctx, x.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + /// `simd_shuffle(x, lane)`. + pub fn simdShuffle(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_shuffle", x, lane); + } + /// `simd_shuffle_xor(x, mask)`. + pub fn simdShuffleXor(self: *Builder, x: Value, mask: anytype) Value { + return self.shuffle("simd_shuffle_xor", x, mask); + } + /// `simd_shuffle_down(x, delta)`. + pub fn simdShuffleDown(self: *Builder, x: Value, delta: anytype) Value { + return self.shuffle("simd_shuffle_down", x, delta); + } + /// `simd_broadcast(x, lane)`. + pub fn simdBroadcast(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_broadcast", x, lane); + } + pub fn simdAny(self: *Builder, x: Value) Value { + return self.emit(air.simd_any(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + pub fn simdAll(self: *Builder, x: Value) Value { + return self.emit(air.simd_all(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + + /// `threadgroup_barrier(flags)`. + pub fn threadgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.wg_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_barrier(flags)`. + pub fn simdgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.simdgroup_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + pub const MatrixOpts = struct { + /// `matrix_origin`. + origin: [2]i64 = .{ 0, 0 }, + /// `transpose_matrix`. + transpose: bool = false, + }; + + fn matrixType(self: *Builder, scalar: Scalar) *const mlir.Type { + return .vector(&.{64}, scalar.mlirType(self.ctx)); + } + + fn matrixScalar(m: Value) Scalar { + return switch (m.kind) { + .matrix => |s| s, + else => std.debug.panic("dsl/metal: expected a simdgroup_matrix, got a {t}", .{m.kind}), + }; + } + + /// The extents, strides and origin a matrix load or store passes the + /// builtin, from its `elements_per_row`, `matrix_origin` and + /// `transpose_matrix`. + fn accessLayout(self: *Builder, elements_per_row: anytype, opts: MatrixOpts) [3]*const mlir.Value { + const ulong2: Type = .vec(.ulong, 2); + const epr = self.convert(self.lift(elements_per_row), .ulong); + const eight = self.constant(.ulong, 8); + const one = self.constant(.ulong, 1); + const o0 = self.constant(.ulong, opts.origin[0]); + const o1 = self.constant(.ulong, opts.origin[1]); + const layout: [3]Value = if (opts.transpose) + .{ self.vec(ulong2, .{ eight, epr }), self.vec(ulong2, .{ epr, one }), self.vec(ulong2, .{ o1, o0 }) } + else + .{ self.vec(ulong2, .{ epr, eight }), self.vec(ulong2, .{ one, epr }), self.vec(ulong2, .{ o0, o1 }) }; + return .{ layout[0].inner, layout[1].inner, layout[2].inner }; + } + + /// `simdgroup_load(m, src, elements_per_row, matrix_origin, + /// transpose_matrix)`: the matrix `m` it loads. + pub fn simdgroupLoad(self: *Builder, src_: Value, elements_per_row: anytype, opts: MatrixOpts) Value { + const src = src_.rvalue(); + const t = src.pointer().pointee.scalar; + const layout = self.accessLayout(elements_per_row, opts); + return self.emit(air.simdgroup_matrix_8x8_load(self.ctx, src.inner, layout[0], layout[1], layout[2], self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// `simdgroup_store(m, dst, elements_per_row, matrix_origin, + /// transpose_matrix)`. + pub fn simdgroupStore(self: *Builder, m: Value, dst_: Value, elements_per_row: anytype, opts: MatrixOpts) void { + const t = matrixScalar(m); + const dst = dst_.rvalue(); + const p = dst.pointer(); + if (p.pointee.scalar != t) std.debug.panic("dsl/metal: simdgroup_store to a {s} pointer", .{p.pointee.name()}); + const layout = self.accessLayout(elements_per_row, opts); + _ = air.simdgroup_matrix_8x8_store(self.ctx, m.inner, dst.inner, layout[0], layout[1], layout[2], self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_multiply_accumulate(d, a, b, c)`: the matrix `d = a * b + c`, + /// of `c`'s type. + pub fn simdgroupMultiplyAccumulate(self: *Builder, a: Value, b: Value, c: Value) Value { + _ = .{ matrixScalar(a), matrixScalar(b) }; + const t = matrixScalar(c); + return self.emit(air.simdgroup_matrix_8x8_multiply_accumulate(self.ctx, a.inner, b.inner, c.inner, self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// The `llvm.loop` metadata of an MSL loop: C++ loops must make progress. + fn loopAnnotation(self: *Builder, opts: LoopOpts) *const llvm.LoopAnnotationAttr { + const unroll: ?*const llvm.LoopUnrollAttr = if (opts.unroll) |u| switch (u) { + .full => llvm.LoopUnrollAttr.get(self.ctx, .{ .full = true }), + .enable => llvm.LoopUnrollAttr.get(self.ctx, .{ .disable = false }), + } else null; + return llvm.LoopAnnotationAttr.get(self.ctx, .{ .unroll = unroll, .mustProgress = true }); + } + + /// The type `i < upper` compares in, for a loop variable `i` of type `t`: the + /// loop runs in `t`, which a short or char variable may be narrower than. + fn loopComparison(t: Type, upper: Type) Scalar { + const compared = types.commonScalar(t.scalar, upper.scalar); + const narrow = t.scalar.bits() < 32 and compared == .int; + if (t.scalar.isFloat() or t.isVector() or (compared.bits() != t.scalar.bits() and !narrow)) + std.debug.panic("dsl/metal: a {s} loop variable compared to a {s}", .{ t.name(), upper.name() }); + return compared; + } + + /// `for (T i = lower; i < upper; i += step)`, with `T` the type of + /// `lower`; `close` ends the body. + pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype) ForScope { + return self.openForOpts(lower, upper, step, .{}); + } + + /// `openFor` under a loop pragma. + pub fn openForOpts(self: *Builder, lower: anytype, upper: anytype, step: anytype, opts: LoopOpts) ForScope { + const i = self.local(if (@TypeOf(lower) == Value) lower.type_() else literalType(lower), lower); + var loop = self.openLoop(opts); + _ = loopComparison(i.type_(), if (@TypeOf(upper) == Value) upper.type_() else literalType(upper)); + loop.body(i.lt(upper)); + const step_: ForScope.Step = switch (@typeInfo(@TypeOf(step))) { + .comptime_int, .int => if (step == 1) .one else .{ .int = step }, + else => .{ .value = step }, + }; + return .{ .loop = loop, .iv = i, .step = step_ }; + } + + /// A loop: `while (c)`, `for (init; c; inc)` or `for (;;)`. + pub fn openLoop(self: *Builder, opts: LoopOpts) LoopScope { + const cond_block = mlir.Block.init(&.{}, &.{}); + _ = self.branch(cond_block); + self.startBlock(cond_block); + return .{ .kernel = self, .cond_block = cond_block, .end_block = mlir.Block.init(&.{}, &.{}), .loop_annotation = self.loopAnnotation(opts) }; + } + + /// `if (cond) { ... }`. + pub fn openIf(self: *Builder, cond: Value) IfScope { + var c = self.condition(); + return c.openIf(cond); + } + + /// `if (cond) { ... } else { ... }` with no result types; with one, `T`, + /// `cond ? a : b` of type `T` evaluating one operand. + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfElseScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + if (comptime common.arity(@TypeOf(result_types), "openIfElse: result_types") == 0) { + var c = self.condition(); + return c.openIfElse(cond, .{}); + } + return self.openTernary(cond, typeOf(result_types[0])); + } + + /// A condition of `&&`, `||` and `!` terms. + pub fn condition(self: *Builder) Condition { + return .{ .kernel = self, .true_block = mlir.Block.init(&.{}, &.{}), .false_block = mlir.Block.init(&.{}, &.{}) }; + } + + /// `a && b && ...` as a bool. + pub fn openAnd(self: *Builder) Logical { + return self.logicalChain(.@"and"); + } + + /// `a || b || ...` as a bool. + pub fn openOr(self: *Builder) Logical { + return self.logicalChain(.@"or"); + } + + fn logicalChain(self: *Builder, op: @FieldType(Logical, "op")) Logical { + var loc_ = [1]*const mlir.Location{self.loc()}; + return .{ .kernel = self, .op = op, .end_block = mlir.Block.init(&.{Type.bool.mlirType(self.ctx)}, &loc_) }; + } + + /// `return;`. + pub fn return_(self: *Builder) void { + const block = self.insert orelse return; + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(block); + self.insert = null; + } + + /// `if (cond) return;`. + pub fn returnIf(self: *Builder, cond: Value) void { + var scope = self.openIf(cond); + { + self.return_(); + scope.close(); + } + } + + /// Terminates the kernel and returns its module, verified. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + self.return_(); + if (self.fast_math) air.enableFastMath(self.ctx, self.module.operation()); + try common.verify(self.module, "dsl/metal"); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, writer.written(), 0); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(ir: []const u8, needles: []const []const u8) !void { + for (needles) |needle| { + if (std.mem.find(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestExpectedEqual; + } + } +} + +test "arguments and C++ arithmetic" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .bfloat, .const_ = true, .restrict = true } }, + .y = .{ .buffer = .{ .type = .float4 } }, + .n = .{ .constant = .int }, + .shmem = .{ .threadgroup = .half }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // `gid.z / n` divides unsigned, `lane * 2` promotes to int. + const q = a.gid.z().div(a.n); + try std.testing.expect(q.type_().eql(.uint)); + const twice = a.lane.mul(2); + try std.testing.expect(twice.type_().eql(.int)); + const v = a.x.load(q).to(.float).mul(0.5); + try std.testing.expect(v.type_().eql(.float)); + a.y.store(twice, b.vec(.float4, .{v})); + a.shmem.store(a.lane, v); + try std.testing.expectEqual(4, b.arguments.items.len); + try std.testing.expectEqual(1, b.arguments.items[1].index); + try std.testing.expectEqual(4, b.arguments.items[2].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_math_enable", "air.kernel", "noalias = true" }); + try expectContains(ir, &.{ "arith.divui", "overflow", "arith.extf", "arith.mulf", "fastmath", "arith.truncf", "llvm.getelementptr", "#air.builtin", "air.arg_name = \"lane\"" }); +} + +test "conversions follow Apple's compiler" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .h = .{ .buffer = .{ .type = .half4, .const_ = true } }, + .o = .{ .buffer = .{ .type = .bfloat4 } }, + .i = .{ .buffer = .{ .type = .int } }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + const f4 = a.h.load(a.tid).to(.float4); + a.o.store(a.tid, f4.to(.bfloat4)); + const n = a.i.load(a.tid); + a.i.store(0, n.to(.float).add(n.to(.half).to(.float)).to(.uint).to(.short)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.convert", ": vector<4xf16> to vector<4xf32>", ": vector<4xf32> to vector<4xbf16>", "air.convert.s", ": i32 to f32", ": i32 to f16", "arith.extf", "air.convert.u", "arith.trunci" }); +} + +test "math, SIMD-group and barrier functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .v = .{ .buffer = .{ .type = .float4 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const x = a.x.load(a.lane); + const v = a.v.load(a.lane); + const m = b.simdMax(b.max(x, b.fast().exp2(x))); + const s = b.simdSum(b.exp(x.sub(m))); + const t = b.precise().tanh(b.fma(s, 2.0, b.dot(v, v))); + const r = b.simdShuffleXor(t, 8); + b.simdgroupBarrier(.mem_threadgroup); + b.threadgroupBarrier(.{ .device = true, .threadgroup = true }); + a.x.store(a.lane, b.ternary(r.gt(0), b.min(a.lane.to(.int), 3).to(.float), r)); + a.x.store(1, b.ternary(r.gt(1), 1.0, 2.0)); + a.v.store(a.lane, b.simdSum(v)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_exp2", "air.fast_fmax", "air.simd_max", "air.fast_exp", "air.simd_sum", "air.dot", "air.fma", "air.tanh", "air.simd_shuffle_xor", "air.simdgroup.barrier ", "air.wg.barrier ", "air.min.s", "arith.select", ": vector<4xf32>" }); + try std.testing.expect(std.mem.find(u8, ir, "cf.cond_br") == null); +} + +test "loops, ifs and early returns" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + var ret = b.openIf(a.tid.ge(1024)); + { + a.x.store(0, 0.0); + b.return_(); + ret.close(); + } + b.returnIf(a.n.le(0)); + + // float acc = 0; for (int i = 0; i < n; ++i) { if (x[i] < 0) break; acc += x[i]; } + const acc = b.local(.float, 0.0); + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + acc.assign(acc.add(a.x.load(loop.iv))); + loop.close(); + } + + // int lo = 0, hi = n; while (lo < hi) { int mid = (lo + hi + 1) / 2; ... } + const lo = b.local(.int, 0); + const hi = b.local(.int, a.n); + var search = b.openLoop(.{}); + { + search.body(lo.lt(hi)); + } + { + const mid = b.local(.int, lo.add(hi).add(1).div(2)); + var up = b.openIfElse(a.x.load(mid).le(acc), .{}); + { + lo.assign(mid); + up.elseBranch(); + } + { + hi.assign(mid.sub(1)); + up.close(); + } + search.close(); + } + + // bool m = lo > 0 || (n > 3 && acc < 1); + var m = b.openOr(); + m.term(lo.gt(0)); + var c = b.condition(); + c.and_(a.n.gt(3)); + var only = c.openIfNot(m.close(acc.lt(1.0))); + { + a.x.store(1, b.ternary(lo.gt(0), acc, 2.0)); + only.close(); + } + + const scratch = b.array(.float, 8); + scratch.store(a.tid.rem(8), acc); + a.x.store(2, scratch.load(3)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "cf.cond_br", "llvm.br", "loop_annotation", "mustProgress = true", "llvm.alloca", "!llvm.array<8 x f32>", "arith.remui" }); + try std.testing.expect(std.mem.find(u8, ir, "scf.") == null); + // The two early returns and the last one, each where it is. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + // Every local is allocated in the entry block, before any branch. + try std.testing.expect(std.mem.findLast(u8, ir, "llvm.alloca").? < std.mem.find(u8, ir, "cf.cond_br").?); +} + +test "a return inside a loop and in both branches of an if" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (;;) { if (n <= 0) break; if (x[0] < 0) return; x[0] -= 1; } + var forever = b.openLoop(.{}); + { + var stop = b.openIf(a.n.le(0)); + { + forever.breakLoop(); + stop.close(); + } + b.returnIf(a.x.load(0).lt(0.0)); + a.x.store(0, a.x.load(0).sub(1.0)); + forever.close(); + } + // if (n > 1) { x[1] = 1; return; } else { x[1] = 2; return; } + var both = b.openIfElse(a.n.gt(1), .{}); + { + a.x.store(1, 1.0); + b.return_(); + both.elseBranch(); + } + { + a.x.store(1, 2.0); + b.return_(); + both.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // The loop's return and one per branch; `finish` adds none after them. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + try expectContains(ir, &.{"loop_annotation"}); +} + +test "a break and the end of the body meet in the latch" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (int i = 0; i < n; ++i) { if (x[i] < 0) break; x[i] = 1; } + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + a.x.store(loop.iv, 1.0); + loop.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i`, and the flag the break and the end of the body set for the latch. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.alloca")); + // `i < n`, `x[i] < 0`, and the latch's choice between leaving and `++i`. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "cf.cond_br")); +} + +test "simdgroup matrices" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float2 } }, + .s = .{ .threadgroup = .bfloat }, + .f = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + const k = b.simdgroupLoad(a.s.add(8), 40, .{ .transpose = true }); + const q = b.simdgroupLoad(a.s, a.f, .{}); + const acc = b.simdgroupMultiplyAccumulate(q, k, b.matrix(.float, a.o.load(a.lane))); + a.o.store(a.lane, acc.threadElements()); + b.simdgroupStore(acc, a.s.to(.float), 8, .{}); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.simdgroup_matrix_8x8_load", "-> vector<64xbf16>", "air.simdgroup_matrix_8x8_multiply_accumulate", "vector<64xbf16>, vector<64xbf16>, vector<64xf32> -> vector<64xf32>", "air.simdgroup_matrix_8x8_store" }); + // `b.matrix` widens two elements to a matrix, `threadElements` takes them back. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.shufflevector")); + try expectContains(ir, &.{"[0, 1] : vector<64xf32>"}); + try std.testing.expect(std.mem.find(u8, ir, "llvm.alloca") == null); +} + +test "constant struct arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{ .fast_math = false }); + defer b.deinit(); + const Args: Struct = .{ .name = "args_t", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .o = .{ .buffer = .{ .type = .float } }, + }); + a.o.store(0, a.args.field("scale").mul(a.args.field("num_heads"))); + try std.testing.expectEqual(8, b.arguments.items[0].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "", "typeName = \"args_t\"", "!llvm.struct<\"struct.args_t\", (i32, f32)>" }); + try std.testing.expect(std.mem.find(u8, ir, "fastmath") == null); +} + +fn expectCount(ir: []const u8, needle: []const u8, expected: usize) !void { + const n = std.mem.count(u8, ir, needle); + if (n != expected) { + std.debug.print("`{s}` {d} times, not {d}, in:\n{s}\n", .{ needle, n, expected, ir }); + return error.TestExpectedEqual; + } +} + +test "subscripts index with the index's type, swizzles with an int" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .v = .{ .buffer = .{ .type = .float4 } }, + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .long }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // float4 x = v[0]; + // o[gid.z] = x.x + x.y + x.z + x.w + x[2] + x[lane] + x[(char) lane] + x[n]; + const x = b.local(.float4, a.v.load(0)); + const sum = x.x().add(x.y()).add(x.z()).add(x.w()).add(x.at(2)).add(x.at(a.lane)).add(x.at(a.lane.to(.char))).add(x.at(a.n)); + a.o.store(a.gid.z(), sum); + // x[lane] = 1.0f; x[2] = 2.0f; x.y = 3.0f; v[0] = x; + x.assign(x.insert(a.lane, 1.0)); + x.assign(x.insert(2, 2.0)); + x.assign(x.insert(1, 3.0)); + a.v.store(0, x); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i32` for the swizzles and literals, `i16 %lane`, `i8`, `i64 %n`. + try expectCount(ir, ": i32] : vector<4xf32>", 7); + try expectCount(ir, ": i16] : vector<4xf32>", 2); + try expectCount(ir, ": i8] : vector<4xf32>", 1); + try expectCount(ir, ": i64] : vector<4xf32>", 1); + try expectCount(ir, ": i32] : vector<3xi32>", 1); +} + +test "a pointer local" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // device float *p = o + 4; p += lane; *p = 1.0f; + const p = b.localPointer(a.o.add(4)); + p.assign(p.add(a.lane)); + p.store(0, 1.0); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectCount(ir, "llvm.alloca", 1); + try expectCount(ir, "llvm.getelementptr", 2); +} + +test "integer max, min, abs and clamp read locals and constant arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .f = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .u = .{ .constant = .uint }, + }); + // int m = n; o[0] = max(m, n) + min(n, m) + min(m, 7) + abs(m) + clamp(m, 0, 7); + const m = b.local(.int, a.n); + a.o.store(0, b.max(m, a.n).add(b.min(a.n, m)).add(b.min(m, 7)).add(b.abs(m)).add(b.clamp(m, 0, 7))); + // w[0] = max(u, 3u) + min(3u, u) + abs(u) + clamp(u, 1u, 2u); + a.w.store(0, b.max(a.u, 3).add(b.min(3, a.u)).add(b.abs(a.u)).add(b.clamp(a.u, 1, 2))); + // const float x = f[0]; f[1] = clamp(x, 0.0f, 1.0f) + fabs(x) + max(x, x); + const x = a.f.load(0); + a.f.store(1, b.clamp(x, 0.0, 1.0).add(b.abs(x)).add(b.max(x, x))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.max.s", "air.min.s", "air.abs.s", "air.clamp.s", "air.max.u", "air.min.u", "air.clamp.u", "air.fast_clamp", "air.fast_fabs", "air.fast_fmax" }); + // abs of a uint is the value itself. + try std.testing.expect(std.mem.find(u8, ir, "air.abs.u") == null); +} + +test "char and uchar take the same functions as wider integers" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .c = .{ .buffer = .{ .type = .char } }, + .u = .{ .buffer = .{ .type = .uchar } }, + .v = .{ .buffer = .{ .type = .char2 } }, + .f = .{ .buffer = .{ .type = .float } }, + .w = .{ .buffer = .{ .type = .float2 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const c = a.c.load(a.lane); + const u = a.u.load(a.lane); + const v = a.v.load(a.lane); + // f[0] = float(c) + float(u); c[0] = char(f[1]); u[0] = uchar(f[2]); + a.f.store(0, c.to(.float).add(u.to(.float))); + a.c.store(0, a.f.load(1).to(.char)); + a.u.store(0, a.f.load(2).to(.uchar)); + // w[0] = float2(v); v[0] = char2(w[1]); + a.w.store(0, v.to(.float2)); + a.v.store(0, a.w.load(1).to(.char2)); + // c[1] = max(c, c); u[1] = min(u, u); c[2] = abs(c); c[3] = clamp(c, 1, 5); v[1] = max(v, v); + a.c.store(1, b.max(c, c)); + a.u.store(1, b.min(u, u)); + a.c.store(2, b.abs(c)); + a.c.store(3, b.clamp(c, 1, 5)); + a.v.store(1, b.max(v, v)); + // c[4] = simd_shuffle(c, lane); c[5] = simd_sum(c); u[2] = simd_max(u); + a.c.store(4, b.simdShuffle(c, a.lane)); + a.c.store(5, b.simdSum(c)); + a.u.store(2, b.simdMax(u)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.convert.s", ": i8 to f32", "air.convert.u", ": f32 to i8", + ": vector<2xi8> to vector<2xf32>", ": vector<2xf32> to vector<2xi8>", "air.max.s", "air.min.u", + "air.abs.s", "air.clamp.s", "air.simd_shuffle.s", "air.simd_sum.s", + "air.simd_max.u", + }); +} + +test "bitwise operators and shifts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .l = .{ .buffer = .{ .type = .long } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // int s = n; o[0] = (n << s) ^ (~(n | 3) & -s); w[0] = w[1] >> s; o[1] = lane >> 1; + // l[0] = l[1] << n; + const s = b.local(.int, a.n); + a.o.store(0, a.n.shl(s).bitXor(a.n.bitOr(3).bitNot().bitAnd(s.neg()))); + a.w.store(0, a.w.load(1).shr(s)); + a.o.store(1, a.lane.shr(1)); + a.l.store(0, a.l.load(1).shl(a.n)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "arith.shli", "arith.shrui", "arith.shrsi", "arith.xori", "arith.ori", "arith.subi" }); + // clang masks an amount to the width (`and i32 %s, 31`), after casting it + // as unsigned (`zext i32 %n to i64`, `and i64 ..., 63`); a number's mask + // folds away. + try expectCount(ir, "arith.andi", 4); + try expectContains(ir, &.{ "arith.constant 31 : i32", "arith.constant 63 : i64" }); + var lines = std.mem.splitScalar(u8, ir, '\n'); + const unsigned_cast = while (lines.next()) |line| { + if (std.mem.find(u8, line, "arith.extui") != null and std.mem.find(u8, line, "i32 to i64") != null) break true; + } else false; + try std.testing.expect(unsigned_cast); +} + +test "a shift reads its amount once" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .n = .{ .constant = .int }, + }); + // int s = n; *o = s << s; + const s = b.local(.int, a.n); + a.o.store(0, s.shl(s)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // n, and s twice. + try expectCount(ir, "llvm.load", 3); +} + +test "math functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .h = .{ .buffer = .{ .type = .half } }, + }); + const v = a.x.load(0); + var sum = b.local(.float, 0.0); + for ([_]Value{ + b.exp2(v), b.log(v), b.log2(v), b.tanh(v), b.sqrt(v), + b.rsqrt(v), b.ceil(v), b.floor(v), b.round(v), b.trunc(v), + b.sin(v), b.cos(v), b.pow(v, 2.0), b.fmod(v, 3.0), + }) |term| sum.assign(sum.add(term)); + const fast_ = b.fast(); + for ([_]Value{ + fast_.exp(v), fast_.log(v), fast_.log2(v), fast_.sqrt(v), fast_.rsqrt(v), + fast_.sin(v), fast_.cos(v), fast_.tanh(v), fast_.max(v, 1.0), fast_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + const precise_ = b.precise(); + for ([_]Value{ + precise_.exp(v), precise_.exp2(v), precise_.log(v), precise_.log2(v), precise_.sqrt(v), + precise_.rsqrt(v), precise_.sin(v), precise_.cos(v), precise_.max(v, 1.0), precise_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + a.x.store(1, sum); + // half has no fast:: functions: exp of a half is precise. + a.h.store(0, b.exp(a.h.load(1))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.fast_exp2", "air.fast_log", "air.fast_log2", "air.fast_tanh", "air.fast_sqrt", "air.fast_rsqrt", + "air.fast_ceil", "air.fast_floor", "air.fast_round", "air.fast_trunc", "air.fast_sin", "air.fast_cos", + "air.fast_pow", "air.fast_fmod", "air.fast_exp", "air.fast_fmax", "air.fast_fmin", + }); + try expectContains(ir, &.{ "air.exp ", "air.exp2 ", "air.log ", "air.log2 ", "air.sqrt ", "air.rsqrt ", "air.sin ", "air.cos ", "air.fmax ", "air.fmin " }); + try expectContains(ir, &.{"f16"}); +} + +test "SIMD-group functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .i = .{ .buffer = .{ .type = .int } }, + .u = .{ .buffer = .{ .type = .uint } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const f = a.x.load(a.lane); + const n = a.i.load(a.lane); + const m = a.u.load(a.lane); + a.x.store(0, b.simdMin(f).add(b.simdPrefixExclusiveSum(f)).add(b.simdShuffle(f, a.lane)).add(b.simdShuffleDown(f, 1)).add(b.simdBroadcast(f, 0))); + a.i.store(0, b.simdMin(n).add(b.simdShuffle(n, 3)).add(b.simdShuffleDown(n, a.lane)).add(b.simdBroadcast(n, 0))); + a.u.store(0, b.simdMin(m).add(b.simdShuffleXor(m, 1))); + a.i.store(1, b.ternary(b.simdAny(f.gt(0.0)), 1, 0).add(b.ternary(b.simdAll(n.ne(0)), 2, 0))); + b.threadgroupBarrier(.mem_device); + b.simdgroupBarrier(.mem_texture); + b.threadgroupBarrier(.mem_none); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.simd_min ", "air.simd_min.s", "air.simd_min.u", "air.simd_prefix_exclusive_sum", + "air.simd_shuffle ", "air.simd_shuffle.s", "air.simd_shuffle_down ", "air.simd_shuffle_down.s", + "air.simd_broadcast ", "air.simd_broadcast.s", "air.simd_shuffle_xor.u", "air.simd_any", + "air.simd_all", "llvm.freeze", + }); + try expectContains(ir, &.{ "air.wg.barrier ", "air.simdgroup.barrier ", "air.wg.barrier " }); +} + +test "pragmas, conditions and casts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + + const r = a.o.load(a.lane).mul(2.0); + // float acc[4] = {}; + const acc = b.zeroedArray(.float, 4); + // #pragma clang loop unroll(full) + var full = b.openForOpts(0, 4, 1, .{ .unroll = .full }); + { + acc.store(full.iv, r); + full.close(); + } + // #pragma unroll + var enabled = b.openForOpts(0, a.n, 2, .{ .unroll = .enable }); + { + acc.store(0, acc.load(0).add(enabled.iv.to(.float))); + enabled.close(); + } + // for (int i = 0; i < 4; i += 1) {} + const i = b.local(.int, 0); + var counted = b.openLoop(.{}); + { + counted.body(i.lt(4)); + } + { + counted.next(); + i.assign(i.add(1)); + counted.close(); + } + // const uint bits = as_type(acc[0]); + const bits = acc.load(0).asType(.uint); + // float t = r; *&t = 2.0f; + const t = b.local(.float, r); + t.address().store(0, 2.0); + // bool ok = n > 0 && bits != 0; + var both = b.openAnd(); + both.term(a.n.gt(0)); + const ok = both.close(bits.ne(0)); + // if (!(n > 1) || ok) { ... } else { ... } + var c1 = b.condition(); + c1.orNot(a.n.gt(1)); + var s1 = c1.openIfElse(ok, .{}); + { + a.o.store(1, 1.0); + s1.elseBranch(); + } + { + a.o.store(1, 2.0); + s1.close(); + } + // if (!ok && !(n == 3)) { ... } else { ... } + var c2 = b.condition(); + c2.andNot(ok); + var s2 = c2.openIfElseNot(a.n.eq(3), .{}); + { + a.o.store(2, 1.0); + s2.elseBranch(); + } + { + s2.close(); + } + // o[lane] = n > 2 ? fast::exp(t) : t; + var choice = b.openIfElse(a.n.gt(2), .{.float}); + { + choice.yieldThen(.{b.fast().exp(t)}); + } + { + choice.yieldElse(.{t}); + } + a.o.store(a.lane, choice.results[0]); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "llvm.memset.p0.i64", "full = true", "disable = false", "llvm.bitcast" }); +} diff --git a/dsl/metal/types.zig b/dsl/metal/types.zig new file mode 100644 index 00000000..9c3072bb --- /dev/null +++ b/dsl/metal/types.zig @@ -0,0 +1,291 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// An MSL scalar type. +pub const Scalar = enum { + bool, + char, + uchar, + short, + ushort, + int, + uint, + long, + ulong, + half, + bfloat, + float, + + pub fn isFloat(self: Scalar) bool { + return switch (self) { + .half, .bfloat, .float => true, + else => false, + }; + } + + pub fn isUnsigned(self: Scalar) bool { + return switch (self) { + .bool, .uchar, .ushort, .uint, .ulong => true, + else => false, + }; + } + + pub fn bits(self: Scalar) u16 { + return switch (self) { + .bool => 1, + .char, .uchar => 8, + .short, .ushort, .half, .bfloat => 16, + .int, .uint, .float => 32, + .long, .ulong => 64, + }; + } + + /// The size in memory: a bool takes a byte. + pub fn size(self: Scalar) u64 { + return if (self == .bool) 1 else self.bits() / 8; + } + + /// The C++ integer conversion rank. + fn rank(self: Scalar) u8 { + return switch (self) { + .bool => 0, + .char, .uchar => 1, + .short, .ushort => 2, + .int, .uint => 3, + .long, .ulong => 4, + else => unreachable, + }; + } + + fn unsignedOf(self: Scalar) Scalar { + return switch (self) { + .char => .uchar, + .short => .ushort, + .int => .uint, + .long => .ulong, + else => self, + }; + } + + /// C++ integral promotion: types narrower than int become int. + pub fn promoted(self: Scalar) Scalar { + return switch (self) { + .bool, .char, .uchar, .short, .ushort => .int, + else => self, + }; + } + + pub fn mlirType(self: Scalar, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .char, .uchar => .int(ctx, .i8), + .short, .ushort => .int(ctx, .i16), + .int, .uint => .int(ctx, .i32), + .long, .ulong => .int(ctx, .i64), + .half => .float(ctx, .f16), + .bfloat => .float(ctx, .bf16), + .float => .float(ctx, .f32), + }; + } +}; + +/// The C++ usual arithmetic conversions of two scalar operand types. +pub fn commonScalar(a: Scalar, b: Scalar) Scalar { + if (a.isFloat() or b.isFloat()) { + if (!b.isFloat()) return a; + if (!a.isFloat()) return b; + if (a == b) return a; + if (a == .float or b == .float) return .float; + std.debug.panic("dsl/metal: no common type of {t} and {t}", .{ a, b }); + } + const x = a.promoted(); + const y = b.promoted(); + if (x == y) return x; + if (x.isUnsigned() == y.isUnsigned()) return if (x.rank() >= y.rank()) x else y; + const u = if (x.isUnsigned()) x else y; + const s = if (x.isUnsigned()) y else x; + if (u.rank() >= s.rank()) return u; + if (s.bits() > u.bits()) return s; + return s.unsignedOf(); +} + +/// An MSL scalar or vector type: `float`, `uint3`, `bfloat4`. +pub const Type = struct { + scalar: Scalar, + /// 1 for a scalar, else the vector length (2, 3 or 4). + len: u8 = 1, + + pub const @"bool": Type = .{ .scalar = .bool }; + pub const char: Type = .{ .scalar = .char }; + pub const uchar: Type = .{ .scalar = .uchar }; + pub const short: Type = .{ .scalar = .short }; + pub const ushort: Type = .{ .scalar = .ushort }; + pub const int: Type = .{ .scalar = .int }; + pub const uint: Type = .{ .scalar = .uint }; + pub const long: Type = .{ .scalar = .long }; + pub const ulong: Type = .{ .scalar = .ulong }; + pub const half: Type = .{ .scalar = .half }; + pub const bfloat: Type = .{ .scalar = .bfloat }; + pub const float: Type = .{ .scalar = .float }; + pub const bool2: Type = .vec(.bool, 2); + pub const bool3: Type = .vec(.bool, 3); + pub const bool4: Type = .vec(.bool, 4); + pub const char2: Type = .vec(.char, 2); + pub const char3: Type = .vec(.char, 3); + pub const char4: Type = .vec(.char, 4); + pub const uchar2: Type = .vec(.uchar, 2); + pub const uchar3: Type = .vec(.uchar, 3); + pub const uchar4: Type = .vec(.uchar, 4); + pub const short2: Type = .vec(.short, 2); + pub const short3: Type = .vec(.short, 3); + pub const short4: Type = .vec(.short, 4); + pub const ushort2: Type = .vec(.ushort, 2); + pub const ushort3: Type = .vec(.ushort, 3); + pub const ushort4: Type = .vec(.ushort, 4); + pub const int2: Type = .vec(.int, 2); + pub const int3: Type = .vec(.int, 3); + pub const int4: Type = .vec(.int, 4); + pub const uint2: Type = .vec(.uint, 2); + pub const uint3: Type = .vec(.uint, 3); + pub const uint4: Type = .vec(.uint, 4); + pub const long2: Type = .vec(.long, 2); + pub const long3: Type = .vec(.long, 3); + pub const long4: Type = .vec(.long, 4); + pub const ulong2: Type = .vec(.ulong, 2); + pub const ulong3: Type = .vec(.ulong, 3); + pub const ulong4: Type = .vec(.ulong, 4); + pub const half2: Type = .vec(.half, 2); + pub const half3: Type = .vec(.half, 3); + pub const half4: Type = .vec(.half, 4); + pub const bfloat2: Type = .vec(.bfloat, 2); + pub const bfloat3: Type = .vec(.bfloat, 3); + pub const bfloat4: Type = .vec(.bfloat, 4); + pub const float2: Type = .vec(.float, 2); + pub const float3: Type = .vec(.float, 3); + pub const float4: Type = .vec(.float, 4); + + /// `vec`. + pub fn vec(scalar: Scalar, len: u8) Type { + std.debug.assert(len >= 2 and len <= 4); + return .{ .scalar = scalar, .len = len }; + } + + pub fn isVector(self: Type) bool { + return self.len > 1; + } + + pub fn element(self: Type) Type { + return .{ .scalar = self.scalar }; + } + + /// The same shape with another element type. + pub fn withScalar(self: Type, scalar: Scalar) Type { + return .{ .scalar = scalar, .len = self.len }; + } + + /// The type in memory, where a bool is a byte, as clang stores it. + pub fn memory(self: Type) Type { + return if (self.scalar == .bool) self.withScalar(.uchar) else self; + } + + /// `sizeof`: a 3-element vector takes the space of 4. + pub fn size(self: Type) u64 { + const len: u64 = if (self.len == 3) 4 else self.len; + return len * self.scalar.size(); + } + + /// `alignof`, which is the size for MSL scalars and vectors. + pub fn alignment(self: Type) u64 { + return self.size(); + } + + pub fn eql(self: Type, other: Type) bool { + return self.scalar == other.scalar and self.len == other.len; + } + + /// The MSL spelling, `uint3`. + pub fn name(self: Type) []const u8 { + const names = comptime blk: { + const scalars = @typeInfo(Scalar).@"enum".field_names; + var table: [scalars.len][4][]const u8 = undefined; + for (scalars, 0..) |scalar, i| { + table[i] = .{ scalar, scalar ++ "2", scalar ++ "3", scalar ++ "4" }; + } + break :blk table; + }; + return names[@backingInt(self.scalar)][self.len - 1]; + } + + pub fn mlirType(self: Type, ctx: *mlir.Context) *const mlir.Type { + const scalar = self.scalar.mlirType(ctx); + return if (self.isVector()) .vector(&.{self.len}, scalar) else scalar; + } +}; + +/// A field of a constant buffer struct: `float s[3]` is +/// `.{ .name = "s", .type = .float, .count = 3 }`. +pub const StructField = struct { + name: []const u8, + type: Type, + /// The array length, zero for a non-array field. + count: u32 = 0, +}; + +/// A struct passed by constant reference, laid out as MSL lays it out. +pub const Struct = struct { + name: []const u8, + fields: []const StructField, + + pub fn offset(self: Struct, index: usize) u64 { + var end: u64 = 0; + for (self.fields[0..index]) |field| { + end = std.mem.alignForward(u64, end, field.type.alignment()); + end += field.type.size() * @max(field.count, 1); + } + return std.mem.alignForward(u64, end, self.fields[index].type.alignment()); + } + + pub fn alignment(self: Struct) u64 { + var a: u64 = 1; + for (self.fields) |field| a = @max(a, field.type.alignment()); + return a; + } + + pub fn size(self: Struct) u64 { + const last = self.fields.len - 1; + const end = self.offset(last) + self.fields[last].type.size() * @max(self.fields[last].count, 1); + return std.mem.alignForward(u64, end, self.alignment()); + } + + pub fn fieldIndex(self: Struct, field_name: []const u8) usize { + for (self.fields, 0..) |field, i| if (std.mem.eql(u8, field.name, field_name)) return i; + std.debug.panic("dsl/metal: struct {s} has no field {s}", .{ self.name, field_name }); + } +}; + +test commonScalar { + try std.testing.expectEqual(Scalar.uint, commonScalar(.uint, .int)); + try std.testing.expectEqual(Scalar.int, commonScalar(.ushort, .short)); + try std.testing.expectEqual(Scalar.long, commonScalar(.long, .uint)); + try std.testing.expectEqual(Scalar.ulong, commonScalar(.ulong, .int)); + try std.testing.expectEqual(Scalar.float, commonScalar(.int, .float)); + try std.testing.expectEqual(Scalar.half, commonScalar(.half, .uint)); + try std.testing.expectEqual(Scalar.float, commonScalar(.half, .float)); +} + +test Struct { + const args: Struct = .{ .name = "A", .fields = &.{ + .{ .name = "n", .type = .int }, + .{ .name = "s", .type = .float, .count = 3 }, + .{ .name = "v", .type = .uint2 }, + .{ .name = "t", .type = .short }, + } }; + try std.testing.expectEqual(0, args.offset(0)); + try std.testing.expectEqual(4, args.offset(1)); + try std.testing.expectEqual(16, args.offset(2)); + try std.testing.expectEqual(24, args.offset(3)); + try std.testing.expectEqual(32, args.size()); + try std.testing.expectEqualStrings("bfloat4", Type.bfloat4.name()); + try std.testing.expectEqual(16, Type.float3.size()); +} diff --git a/dsl/mosaic_tpu/BUILD.bazel b/dsl/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..15510258 --- /dev/null +++ b/dsl/mosaic_tpu/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + "pipeline.zig", + ], + import_name = "dsl/mosaic_tpu/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/affine", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/scf", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md new file mode 100644 index 00000000..cb1e0bab --- /dev/null +++ b/dsl/mosaic_tpu/README.md @@ -0,0 +1,846 @@ +# ZML Mosaic TPU DSL — Author's Guide + +A Zig builder for Mosaic TPU IR (the `tpu` dialect in XLA) that tries +to read like `pallas.tpu` Python while staying in pure Zig. Used by +`zml/kernels/moe/mosaic_tpu/`, `zml/kernels/paged_attention/mosaic_tpu/` +and `platforms/tpu/`, and by anything else that wants to emit a TPU kernel +op-for-op against what `pallas_call` produces. + +The DSL builds `tpu` operations and attributes through the bindings in +`mlir/dialects/mosaic_tpu`. +A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway +context, canonicalizes it, serializes it to a string, and hands that to a +`stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's +compilation context (which carries only `func`/`stablehlo`/`sdy`). The dialect and +its C API come from the pinned XLA repository. + +Two layers define the surface: + +- **Low layer** — `dsl/mosaic_tpu/builder.zig`: the IR builder primitives. + `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` + → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` + for the final IR string. Pass `Builder.open` / `Builder.init` a + `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects + the DSL emits), `defer ctx.deinit()` once you have the string — not a + `Compilation`'s `mlir_ctx`. +- **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the + declarative kernel form, mirroring the Triton DSL. Bundles a config + type, a typed spec literal (name + named `inputs` / `outputs` + a typed + `run` function pointer), and produces a kernel type. Exposes + `.emit(allocator, cfg)` (IR string; manages its own context) and + `.call(inputs, outputs, opts)` (IR string + `stablehlo.custom_call` + targeting `tpu_custom_call` in one shot — the preferred form for + production model code). + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [Argument specs](#argument-specs) +3. [Pallas `pallas_call` metadata](#pallas-pallas_call-metadata) +4. [The `Value` type](#the-value-type) +5. [Fluent methods on `Value`](#fluent-methods-on-value) +6. [Polymorphic scalars](#polymorphic-scalars) +7. [Constants and constant hoisting](#constants-and-constant-hoisting) +8. [Loads and stores](#loads-and-stores) +9. [Vector and shape helpers](#vector-and-shape-helpers) +10. [Reductions, scans, sorts](#reductions-scans-sorts) +11. [Compute primitives](#compute-primitives) +12. [Memref ops](#memref-ops) +13. [DMAs, semaphores, and barriers](#dmas-semaphores-and-barriers) +14. [Control flow](#control-flow) +15. [Escape hatches](#escape-hatches) +16. [Pallas Python ↔ Zig DSL cheat-sheet](#pallas-python--zig-dsl-cheat-sheet) +17. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const mlir = @import("mlir"); +const mtt = @import("zml").kernel.mosaic_tpu; + +pub const MyKernel = struct { + pub const Cfg = struct { /* shape/dtype/grid knobs */ }; + pub const Kernel = mtt.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x", "cu_seqlens" }, + .outputs = &.{"y"}, + .run = run, + }); + fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + _ = cfg; + const a = try b.declareArgsOpts(.{ + // Grid program-ids — i32 scalars. `pl.program_id(0)`, `pl.program_id(1)`. + .i_b = .{ .scalar = .i32 }, + + // Inputs: VMEM memrefs. + .x = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16 } }, + + // SMEM scalar-prefetch. + .cu_seqlens = .{ .ref = .{ .shape = &.{4}, .dtype = .i32, .memory_space = .smem } }, + + // Outputs. + .y = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16, .role = .output } }, + }, &.{}, .{ + .dimension_semantics = &.{.arbitrary}, + .iteration_bounds = &.{ /* grid */ 4 }, + .pallas_window_params = true, + }); + _ = a; + + // ... emit body via b.* helpers; reach args through `a.`. + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. +2. **`pub const Kernel = mtt.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body, mirroring the Triton form. First + call `try b.declareArgsOpts(.{…}, &.{}, opts)` to create the `func.func` + op and bind a `NamedArgs(Spec)` struct of `Value`s. The factory has + already opened the builder; it'll terminate, verify, run + `canonicalize,cse,canonicalize`, and serialize after `run` returns — + no manual `b.finish(...)` call. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgsOpts(...)` (which are the **MLIR-level** arg +names — what shows up in the `func.func` signature). By convention they +often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgsOpts(...)` is a struct literal — its +variant *tags* (`.scalar`, `.iter`, `.ref`, `.sem`) are comptime-known per +call site, but the *payload values* (shapes, dtypes, memory spaces, +windows) can be runtime values from a config. + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). +> +> Single-kernel files (e.g. `zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig`) +> can skip the wrapper and put `pub const Cfg` / `pub const Kernel` at +> module scope directly — the wrapper is just convenient when several +> kernels share a file. + +### Two ways to use a kernel + +**Offline / tooling** — get the MLIR string and print or diff it (no MLIR +context needed; `emit` spins up and tears down its own): + +```zig +const ir = try MyKernel.Kernel.emit(allocator, .{ /* cfg */ }); +defer allocator.free(ir); +try stdout.print("{s}\n", .{ir}); +``` + +**Inside a ZML model** — emit IR and drop a `stablehlo.custom_call` +(targeting `tpu_custom_call`) into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x = x_tensor, .cu_seqlens = cu }, // Inputs (by name) + .{ .y = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ /* cfg */ }, + .extras = .{ .vmem_limit_bytes = 1 << 20 }, + }, +).y; +``` + +`Kernel.call` takes a `mosaic_tpu.CallExtras` via `opts.extras` for +TPU-specific knobs (`vmem_limit_bytes`, `disable_bounds_checks`, +`disable_semaphore_checks`, `has_communication`, plus +`additional_attributes` and `output_operand_aliases` for in-place +updates). + +--- + +## Argument specs + +Each field passed to `b.declareArgs(...)` / `b.declareArgsOpts(...)` is an +`ArgSpec.Kind` tagged-union literal. The four kinds mirror what a Mosaic +`func.func` can carry: + +| Kind literal | MLIR type | Use case | +|--------------------------------------------------------|--------------------------------------|-----------------------------------------| +| `.{ .scalar = .i32 }` | plain `i32` | `program_id`, `num_programs`, sizes | +| `.{ .iter = .{ .semantics = .parallel } }` | `index` | Loop / grid iter index with optional `tpu.dimension_semantics` | +| `.{ .ref = .{ .shape = &.{...}, .dtype = .bf16 } }` | `memref<… x T, #tpu.memory_space<...>>` | Pallas-style operand | +| `.{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }` | `memref<…x!tpu.(dma_)semaphore, semaphore_mem>` | Scratch sem pool (e.g. DMA double-buffer) | +| `.{ .sem = .regular }` / `.{ .sem = .dma }` | `!tpu.semaphore` / `!tpu.dma_semaphore` | Single semaphore handle | + +### Roles + +`RefSpec.role` mirrors what Pallas `pallas_call` calls each operand: + +| Role | Where it appears | `window_params` / `transform_N` | +|--------------------|-----------------------------------------------------|---------------------------------| +| `.input` (default) | Inputs declared via `in_specs` | one entry per input | +| `.output` | Outputs declared via `out_specs` | one entry per output | +| `.scalar_prefetch` | Leading SMEM refs (Pallas's `scalar_prefetch`) | none — but appended to every `transform_N` body's args | +| `.scratch` | Trailing scratch refs (kv_bufs, l/m/acc, sem_array) | none | + +The DSL filters `window_params` to only `.input`/`.output` entries (matching +Pallas's `len(grid_mapping.block_mappings)`), and re-emits the +`scalar_prefetch` refs as trailing args on every emitted `transform_N` stub. + +The default `memory_space` for a `.ref` is `.vmem`. Override with `.smem`, +`.cmem`, `.hbm`, `.semaphore_mem`, `.vmem_shared`, or `.any` per the +`MemorySpace` enum. Iteration semantics (`.parallel`, `.arbitrary`, +`.core_parallel`, `.subcore_parallel`) attach `tpu.dimension_semantics` to +the iter arg. + +**Runtime payloads:** the variant tags are fixed per kernel, but shape / +dtype / memory_space values can be runtime — typical for kernels +parameterized by a `Config` struct. + +--- + +## Pallas `pallas_call` metadata + +The wrapper attributes Pallas attaches to the `func.func` are exposed through +`Builder.Opts`: + +| Field | Type | Meaning | +|--------------------------|---------------------------|--------------------------------------------------------------| +| `.@"noinline"` | `bool` (default false) | `no_inline` attribute on the function. | +| `.core_type` | `?CoreType` (default `.tc`) | `tpu.core_type = #tpu.core_type<…>`. | +| `.dimension_semantics` | `[]const DimensionSemantics` | `dimension_semantics = [...]` — one entry per grid dim. | +| `.scalar_prefetch` | `i64` (default 0) | `scalar_prefetch = N : i64`. | +| `.scratch_operands` | `i64` (default 0) | `scratch_operands = N : i64`. | +| `.iteration_bounds` | `?[]const i64` | `iteration_bounds = array` — the grid shape. | +| `.pallas_window_params` | `bool` (default false) | Auto-emit `window_params = [...]` and trailing `transform_N` stub funcs. | + +Per-arg `window_params` behavior follows +`jax/_src/pallas/mosaic/lowering.py:1014–1131` exactly. With +`pallas_window_params = true`: + +- **`.hbm` / `.any` / `.semaphore_mem`** memory spaces ⇒ empty `{}` window + entry, **no `transform_N`**. Anything else gets a populated dict. +- **VMEM trivial-window** (no `RefSpec.window`, or `window.block_shape == ref.shape`) + ⇒ auto-force `pipeline_mode = synchronous` (matching Pallas's + `Buffered(1)` injection). +- **Otherwise** ⇒ emit only attributes the user explicitly set on + `RefSpec.window`. +- **`window_bounds`** = `block_shape` (defaults to operand shape). +- **`transform_N` body** returns `block_rank` i32 zeros, mirroring a trivial + index map. + +Override per ref-arg via `RefSpec.window`: + +```zig +.x = .{ .ref = .{ + .shape = &.{ 256, 512 }, + .dtype = .bf16, + .window = mtt.ArgSpec.WindowSpec{ + .block_shape = &.{ 128, 512 }, + .pipeline_mode = .double_buffered, + .revisit_mode = .any, + .element_window = .{ .pad_low = &.{ 0, 0 }, .pad_high = &.{ 0, 0 } }, + // Custom transform_N return — by default the body returns + // `block_rank` zeros (a trivial index_map). Override per slot. + .transform_returns = &.{ + .{ .program_id = 1 }, // q_blk_idx + .{ .program_id = 0 }, // heads_blk_idx + .zero, + }, + }, +} }, +``` + +The DSL treats the window as **non-trivial** (so it does *not* auto-force +`pipeline_mode = synchronous` for VMEM) when `transform_returns` is non-null +and contains anything other than all `.zero`s. This matches Pallas's +`bm.has_trivial_window()` check. + +`PipelineMode` is `{ synchronous, double_buffered }` (matching Pallas's +`Buffered(1)` / `Buffered(2)`). `RevisitMode` is `{ immediate, any }`. + +> **Anonymous-struct gotcha.** Anonymous-struct literals (`.{ ... }`) don't +> coerce through the optional-pointer wrapping that Zig 0.16 expects for +> `RefSpec.window`. Always write the type explicitly: +> `.window = mtt.ArgSpec.WindowSpec{ ... }`. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when calling raw + `tpu.*` / `arith.*` / `vector.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through + it. The DSL populates `kernel` automatically for every value it creates + (`emit`, `arg`, loop-region args, etc.). If you construct a `Value` by hand + (`.{ .inner = raw }`), the fluent methods will panic — use `Builder.*` + static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|----------------------------------|------------------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar, vector, or memref). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isVector()` | `bool` | `true` for `vector<…>`. | +| `isMemRef()` | `bool` | `true` for `memref<…>`. | +| `isShaped()` | `bool` | Vector or memref. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, MAX_RANK) | Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / … based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | +|----------------|--------------|------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | + +For Python-style floor-division (`a // b` matches `jax.numpy.floor_divide`'s +sign-correcting expansion — `divsi` plus a `(sign(a) != sign(b)) && (a % b != +0)` correction), use `b.divFloor(lhs, rhs)`. `divsi` alone truncates toward +zero, which differs for mixed-sign operands. +| `.bitAnd(rhs)` | `arith.andi` | — | +| `.bitOr(rhs)` | `arith.ori` | — | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | + +### Comparisons + +All return i1 (scalar) or `vector<… x i1>`. Signed integer predicates by +default; ordered float predicates for floats. + +| Method | Int predicate | Float predicate | +|------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts + +| Method | Effect | +|-----------------------|-----------------------------------------------------| +| `.to(dtype)` | Numeric cast preserving shape; auto-dispatches int↔float. | +| `.bitCast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | +| `.abs()` | `math.absf` (float) / `math.absi` (int). | + +`.to(...)` chooses the right MLIR op based on the source vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|----------------|--------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` | +| float | narrower float | `arith.truncf` | +| same type | same type | no-op | + +For `index`-typed values use `b.indexCast(v, target_ty)` or `b.toIndex(v)` +explicitly — `.to(...)` is for numeric types. + +--- + +## Polymorphic scalars + +Most `Builder.*` helpers and every fluent method's RHS take `anytype`. The +lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that + preserves the source Zig width. + +If you need to force a specific width for a runtime variable, cast at the +call site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When do you need `b.lift` / `b.constant` / `b.constIndex`? + +| Helper | Effect | +|--------------------------------|----------------------------------------------------------| +| `b.lift(value)` | Wrap a Zig scalar; dtype inferred from source. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as `like`'s element type. | +| `b.constIndex(value)` | `arith.constant N : index` — for memref offsets. | + +Reach for these when the DSL surface demands an existing `Value` and there's +no binop sibling to match against — typically loop seeds for +`openWhile(.{inits}, ...)`, `openFor(0, N, 1, .{init})`, or `var x: Value = +b.constant(.i32, 0);` that a later branch may reassign. + +If the constant flows through any `.add` / `.mul` / `splat` etc., skip the +lift and let auto-lift do it. + +--- + +## Constants and constant hoisting + +The DSL provides splat-style constructors for the dense-splat fast-path that +Pallas/Mosaic uses: + +| Helper | Emits | +|----------------------------------------|---------------------------------------------| +| `b.zeros(shape, dtype)` | `arith.constant dense<0.0> : vector<…>` | +| `b.ones(shape, dtype)` | Ones vector via dense splat. | +| `b.full(shape, value, dtype)` | `arith.constant dense : vector<…>` | +| `b.splat(value, shape, dtype)` | Same as `full` for comptime values; `vector.broadcast` for runtime Values. | + +These emit a single `arith.constant` (instead of a scalar-const + +`vector.broadcast` pair) so the IR diff vs Pallas stays empty for splat-only +constants. + +--- + +## Loads and stores + +Mosaic separates **vector** loads/stores from **scalar** memref loads/stores. +Helpers infer everything from the memref's type — there is no `result_type` +argument. + +### Vector + +| Helper | Pallas / Mosaic equivalent | +|---------------------------------------------------------|-------------------------------------------------------| +| `b.refLoad(ref)` | `vector.load ref[0, …, 0] : memref<…>, vector<…>` | +| `b.vectorLoadAt(ref, indices)` | `vector.load ref[indices...]` with full ref shape. | +| `b.vectorLoadShape(ref, indices, shape)` | Sliced `vector.load` — `ref[i, j, :, k:k+BLOCK]`. | +| `b.tpuVectorLoad(ref, indices, .{ .strides, .mask })` | `tpu.vector_load` — sublane stride + mask form. | +| `b.refStore(ref, value)` | Full-shape `tpu.vector_store ref[0, …]`. | +| `b.vectorStoreAt(ref, value, indices)` | `tpu.vector_store ref[indices…]`. | +| `b.refStoreOpts(ref, value, indices, .{ .strides, .mask, .add })` | Strided / masked / atomic-add store. | +| `b.stridedLoad(ref, indices, strides)` | `tpu.strided_load` — result shape = `ref.shape / strides` (per-dim ceil). | +| `b.stridedLoadShape(ref, indices, strides, result_shape)` | `tpu.strided_load` with explicit result shape — for cropped loads (e.g. half-rows). | +| `b.stridedStore(ref, value, indices, strides)` | `tpu.strided_store`. | + +> **`refLoad` vs `tpuVectorLoad`.** Pallas's default emit is +> `vector.load`, which is what `refLoad` / `vectorLoadAt` produce. Reach for +> `tpuVectorLoad` only when you need sublane stride or a per-element mask +> (the `tpu.vector_load` form). Same on the store side. + +`indices` must be `index`-typed Values (use `b.toIndex(v)` to convert from +i32). If you pass `&.{}`, the helper pads with a cached `arith.constant 0 : +index` to match the ref's rank — the same SSA value is reused per block, so +no `%c0` duplication shows up in the IR. + +### Scalar (SMEM) + +| Helper | MLIR op | +|-------------------------------------------------|------------------------------------------| +| `b.scalarLoad(ref, indices)` | `memref.load` | +| `b.scalarStore(ref, value, indices)` | `memref.store` | + +Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar +`memref.load`. + +--- + +## Vector and shape helpers + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.broadcastTo(src, shape)` | `vector.broadcast` — leading-dim broadcast or rank-prepend. | +| `b.shapeCast(src, new_shape)` | `vector.shape_cast` — reshape preserving total lanes. | +| `b.reshape(src, new_shape)` | `tpu.reshape` (TPU-flavored reshape; differs from `shapeCast`). | +| `b.repeat(src, dimension, times)` | `tpu.concatenate` of `times` copies, as `pltpu.repeat`. | +| `b.concatenate(sources, dimension)` | `tpu.concatenate`. | +| `b.transpose(src, permutation)` | `tpu.transpose`. | +| `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | +| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.dynamic_rotate` with a constant i32 amount. | +| `b.vectorExtract(src, position)` | `vector.extract` — scalar / lower-rank slice. | +| `b.iota(shape, dtype, dimensions)` | `tpu.iota` over a chosen dim. | +| `b.arange(n, dtype)` | Pallas-style 1-D iota: `<1xN>` `tpu.iota` + `vector.shape_cast` to ``. Mirrors `_iota_lowering_rule`. | + +> **Use `arange`, not `iota(&.{N})`.** The TPU layout pass requires vectors +> to be 2-D with the lane dim trailing; a bare 1-D `tpu.iota` is rejected. +> `arange` emits the `<1xN>` + `shape_cast` pair Pallas's lowering produces. + +> **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream +> `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's +> `tpu.reshape`. Stick to `shapeCast` when matching Pallas IR. + +### Auto-broadcast + +The fluent binary ops and comparisons (`add` … `maximum`, `lt` … `ne`), +`b.where`, `b.maximum` and `b.minimum` splat a scalar operand against a +vector one (`vector.broadcast`) and widen the narrower integer side. Nothing +else broadcasts: Mosaic kernels run inside the TPU's tile semantics and the +layout pass is strict about `vector<…>` ranks/shapes, so build rank changes +and size-1 expansions explicitly with `shapeCast` + `broadcastTo`: + +```zig +// b_k[:, None] broadcast to +const bk_2d = k.shapeCast(b_k, &.{ BK, 1 }); +const bk_bc = k.broadcastTo(bk_2d, &.{ BK, BV }); +``` + +For a vector made from a scalar outside a binary op, use +`b.broadcastTo(scalar, shape)` (it routes through `vector.broadcast`) or +`b.full / b.splat` for comptime constants. + +--- + +## Reductions, scans, sorts + +### Vector-dialect reductions + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.multiReduction(kind, src, acc, axes)` | `vector.multi_reduction` — multi-D reduce along `axes`. | +| `b.reduceToScalar(kind, src, acc)` | 1-D vec → scalar via the Pallas idiom: `shape_cast→<1xN>`, `multi_reduction[1]`, `extract[0]`. | +| `b.vectorReductionFlat(kind, src)` | `vector.reduction` — bare 1-D reduce. Layout pass may reject; prefer `reduceToScalar`. | + +`kind` is a `CombiningKind` (`.add`, `.mul`, `.minsi`, `.maxsi`, `.minnumf`, +`.maxnumf`, `.and_`, `.or_`, `.xor`). + +> **Pallas's reduce-to-scalar idiom.** Pallas never emits a bare +> `vector.reduction` for the L2-norm / sum-to-scalar pattern — it always +> goes through `multi_reduction` after a leading-dim `shape_cast`. The +> `reduceToScalar` wrapper produces exactly that shape, so kernels stay +> byte-equivalent to `pallas_call`'s output. See +> `jax/_src/pallas/mosaic/lowering.py:reduce_lowering_rule`. + +### TPU-dialect reductions / scan / sort + +| Helper | MLIR op | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.allReduce(input, kind, dim)` | `tpu.all_reduce` — keeps `dim` as size 1. | +| `b.reduceIndex(input, kind, axis)` | `tpu.reduce_index` — drop axis, replace with `index`. | +| `b.scan(input, kind, mask)` | `tpu.scan` along the last dimension; optional rank-one mask. | +| `b.sort(keys, values, mask, .{ .descending })` | `tpu.sort` — returns `[3]Value` (sorted_keys, indices, sorted_values). | + +--- + +## Compute primitives + +### Math (scalar / elementwise) + +| Helper | MLIR op | +|------------------------------------------------|--------------------| +| `b.exp(x)` | `math.exp` | +| `b.exp2(x)` | `exp(x * ln2)` — matches Pallas's `_exp2_lowering_rule` (forward-compat). | +| `b.mathExp2(x)` | Raw `math.exp2` — only legal on cloud TPUs after 2025-07-26. | +| `b.log(x)` / `b.log2(x)` | `math.log` / `math.log2` | +| `b.sqrt(x)` / `b.rsqrt(x)` | `math.sqrt` / `math.rsqrt` | +| `b.sin(x)` / `b.cos(x)` / `b.tanh(x)` / `b.erf(x)` | `math.*` | +| `b.absf(x)` / `b.absi(x)` / `b.abs(x)` | `math.absf` / `math.absi` (auto) | +| `b.floor(x)` / `b.ceil(x)` | `math.floor` / `math.ceil` | +| `b.powf(x, y)` | `math.powf` | +| `b.fma(a, b, c)` | `math.fma` | +| `b.reciprocal(x)` / `b.reciprocalOpts(x, .{ .approximate })` | `tpu.reciprocal` | + +### Matmul + +```zig +// y = lhs @ rhs + acc +const y = k.matmul(lhs, rhs, acc); +const y2 = k.matmulOpts(lhs, rhs, acc, .{ + .transpose_lhs = false, + .transpose_rhs = true, + .precision = .bf16, +}); +``` + +`matmul` lowers to `tpu.matmul`; result type is taken from `acc`. Use +`matmulOpts` to set transposes, `tpu.contract_precision`, or a custom +`#tpu.dot_dimension_numbers<…>` attribute. + +### Selects / where + +| Helper | MLIR op | +|---------------------------------|--------------------------| +| `b.select(cond, t, f)` | `arith.select` | +| `b.where(cond, x, y)` | Alias of `select`. | + +### Masks + +| Helper | Effect | +|----------------------------------------------------------|-----------------------------------------| +| `b.createMask(low_bounds, high_bounds, shape)` | `vector.create_mask`. | +| `b.rangeMask(limits, shape)` | `vector.create_mask` from `0..limit`. | + +--- + +## Memref ops + +For carving sub-references out of a larger memref or reinterpreting an +existing one: + +| Helper | MLIR op | +|-----------------------------------------------------------------|--------------------------| +| `b.memRefSlice(mem_ref, offsets, sizes, strides, result_shape)` | `memref.subview`. | +| `b.memRefSqueeze(mem_ref, result_shape)` | `memref.collapse_shape` style. | +| `b.memRefReshape(mem_ref, result_shape)` | `tpu.memref_reshape`. | +| `b.memRefBitcast(mem_ref, dtype)` | `tpu.memref_bitcast` — same shape, new element dtype. | +| `b.memRefBitcastShape(mem_ref, result_shape, dtype)` | `tpu.memref_bitcast` with explicit result shape — for the bf16↔i32 packing trick (`<64x128xbf16>` → `<32x128xi32>`). | +| `b.reinterpretCast(mem_ref, result_shape, dtype)` | `memref.reinterpret_cast`. | +| `b.assumeLayout(src)` | `tpu.assume_layout`. | +| `b.eraseMemRefLayout(mem_ref)` | `memref.cast` to no-layout. | + +--- + +## DMAs, semaphores, and barriers + +For multi-core / async DMA orchestration: + +| Helper | MLIR op | +|------------------------------------------------------------------------------|------------------------| +| `b.semAlloc(.regular)` / `b.semAlloc(.dma)` | `tpu.sem_alloc`. | +| `b.semBarrier()` | `tpu.sem_barrier`. | +| `b.semRead(sem)` | `tpu.sem_read`. | +| `b.semWait(sem, amount)` | `tpu.sem_wait`. | +| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id, .subcore_id })` | `tpu.sem_signal`. | +| `b.barrier(barrier_id)` | `tpu.barrier`. | +| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .subcore_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | +| `b.waitDma2(sem, src, dst, .{ .device_id, .core_id, .strict_ordering })` | `tpu.wait_dma2`. | +| `b.deviceId()` | `tpu.device_id`. | +| `b.delay(cycles)` | `tpu.delay`. | +| `b.traceStart(level, message)` / `b.traceStop()` | `tpu.trace_start` / `tpu.trace_stop` — Pallas's einsum boundaries. | +| `b.dotDimensionNumbers(lhs_c, rhs_c, lhs_nc, rhs_nc, output_order, lhs_b, rhs_b)` | Build a `#tpu.dot_dimension_numbers<...>` attr for `matmulOpts`. | + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed +scope value. You emit body ops into a bare `{}` block, terminate with +`yield` / `yieldThen` / `yieldAfter` (or `close()` for a scope without +values), and read results from `scope.results` afterward. No `ctx` struct, +no function literal — lexical scope carries captures. + +### `openFor` — scf.for with iter_args + +```zig +var loop = k.openFor(c0_i32, T, c1_i32, .{ b_h_init }); +{ + const t = loop.iv; + const b_h = loop.carried[0]; + + // ... body ops ... + + loop.yield(.{ b_h_updated }); +} +const final_b_h = loop.results[0]; +``` + +- `lower` / `upper` / `step` are `anytype`: pass Values, comptime ints, or + runtime Zig ints. A bound that isn't a Value takes the type of the first one + that is; with none, all three are `index`. Pass `b.constant(.i32, 0)` as + `lower` for an `i32` counter (matching Pallas's `scf.for ... : i32`). +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an + array) all have the same length. +- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.close()`. +- `b.openForOpts(lower, upper, step, inits, .{ .unsigned_cmp = true })` sets + the loop's options. + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = k.openIf(cond); +{ + // then body — store, increment, etc. + i.close(); +} +``` + +Use for conditional side-effects (no results). The else block is auto-built +empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = k.openIfElse(cond, .{.f32}); +{ + i.yieldThen(.{ x }); +} +{ + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `k.vectorTy(shape, dtype)` / + `k.memRefTy(shape, dtype, mem_space)`. Pass `.{}` for an if/else with no + results, then end its branches with `elseBranch()` and `close()`. + +### `openWhile` — scf.while + +```zig +var w = k.openWhile(.{ i0 }, .{.i32}); +{ + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); +} +{ + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); +} +const r = w.results[0]; +``` + +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack. +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. +- After any `yield*`, the scope's `.results` holds the scf op's results as + a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `tpu.*` / + `arith.*` / `vector.*` / `memref.*` / `scf.*` / `math.*` constructor + directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its + `kernel` field populated. +- `b.scalarTy(dtype)` / `b.vectorTy(shape, dtype)` / + `b.memRefTy(shape, dtype, mem_space)` — type constructors used by raw + ops and by non-scalar `openIfElse.result_types` / `openWhile.after_types`. + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Pallas Python ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-------------------------------------------------------|--------------------------------------------------------------| +| `pl.program_id(0)` | `a.i_v` (declared as `.{ .scalar = .i32 }`) | +| `cu_seqlens[i]` | `k.scalarLoad(a.cu_seqlens, &.{ k.toIndex(i) })` | +| `lax.div(a, b)` | `k.divsi(a, b)` | +| `lax.rem(a, b)` | `k.remsi(a, b)` | +| `jnp.arange(N, dtype=jnp.int32)` | `k.arange(N, .i32)` | +| `o_k < KEY_DIM` | `k.cmpi(.slt, o_k, dim_splat)` / `o_k.lt(dim_splat)` | +| `mask_k[:, None] & mask_v[None, :]` | `k.andi(k.broadcastTo(k.shapeCast(mask_k, &.{BK,1}), &.{BK,BV}), k.broadcastTo(mask_v, &.{BK,BV}))` | +| `jnp.where(mask, x, 0.0)` | `k.select(mask, x, k.zeros(shape, dtype))` | +| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `k.shapeCast(v, &.{BK})` then `.to(.f32)` | +| `q_ref[0, t, h, :] = b_o.astype(o_ref.dtype)` | `k.vectorStoreAt(a.q, k.shapeCast(b_o, &.{1,1,1,BK}), &.{c0, t_idx, h_idx, c0})` | +| `jnp.sqrt(jnp.sum(b_q * b_q) + 1e-6)` | `k.sqrt(k.addf(k.reduceToScalar(.add, k.mulf(b_q, b_q), cst_zero_1), cst_eps))` | +| `jnp.exp(b_g)` | `k.exp(b_g)` | +| `jnp.sum(b_h * b_k[:, None], axis=0)` | `k.multiReduction(.add, k.mulf(b_h, bk_bc), cst_zero_v, &.{0})` | +| `b_h * jnp.exp(b_g)` (broadcast to ) | `k.mulf(b_h, k.broadcastTo(exp_g, &.{BK, BV}))` | +| `b_beta * (b_v - predicted_v)` | `k.mulf(k.broadcastTo(b_beta, &.{BV}), k.subf(b_v, predicted_v))` | +| `lax.fori_loop(0, T, body, init)` | `var loop = k.openFor(c0_i32, T, c1_i32, .{init}); { …; loop.yield(.{…}); }` | +| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.close(); }` | +| `if cond: x = a else: x = b` | `var i = k.openIfElse(cond, .{ty}); { …; i.yieldThen(.{a}); }; { …; i.yieldElse(.{b}); }` | +| `while cond:` | `var w = k.openWhile(.{inits}, .{after_types}); { …; w.yieldBefore(cond, .{…}); }; { …; w.yieldAfter(.{…}); }` | +| `pltpu.MemorySpace.SMEM` | `.memory_space = .smem` on the `RefSpec` | +| `pl.BlockSpec(memory_space=…)` (trivial window) | Default `RefSpec` — auto-`synchronous` on VMEM | +| `pl.BlockSpec(block_shape=(BM, BK), …)` | `.window = .{ .block_shape = &.{BM, BK} }` | +| `pl.BlockSpec(..., pipeline_mode=Buffered(2))` | `.window = .{ .pipeline_mode = .double_buffered }` | + +--- + +## Pitfalls + +1. **TPU vectors are 2-D minimum.** A bare `tpu.iota : vector` or + `vector.reduction` on `vector` is rejected by the layout pass. + Use `b.arange(N, dtype)` (emits the `<1xN>` iota + `shape_cast`) and + `b.reduceToScalar(...)` (emits the `shape_cast→<1xN>` + + `multi_reduction[1]` + `extract[0]` triple). Match Pallas's emit pattern + verbatim and the layout pass is happy. + +2. **No implicit auto-broadcast.** Unlike the Triton DSL, Mosaic won't + silently `expand_dims` / `broadcast` a rank-mismatched RHS. Build the + broadcasts explicitly (`shapeCast` + `broadcastTo`). The Pallas Python + DSL emits the same explicit broadcast pair — mirroring it keeps the IR + diff empty. + +3. **`vector.load` indices must be `index`-typed.** Convert from i32 with + `k.toIndex(v)` (emits `arith.index_cast`). JAX does no CSE inside + `pallas_call` lowering, so re-emitting the same `index_cast` per use is + the byte-equivalent thing to do. + +4. **Re-emit derived offsets per use.** Pallas re-runs its index_map + computation per operand load/store rather than CSE-ing intermediate + `arith.muli` / `arith.addi`. To match its IR exactly, do the same: emit + `iv_off = i_v * BV` again at every site that needs it. The MLIR + canonicalizer collapses duplicates downstream. + +5. **`refLoad` vs `tpuVectorLoad`.** Pallas's default `ref[...]` lowers to + `vector.load` (not `tpu.vector_load`). Use `refLoad` / `vectorLoadAt` / + `vectorLoadShape` for byte-equivalence; reach for `tpuVectorLoad` only + when you need `tpu.vector_load`-specific features (sublane stride or + per-element mask). + +6. **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream + `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's + `tpu.reshape` (different op, different layout semantics). Stick with + `shapeCast` when matching Pallas IR. + +7. **SMEM args take scalar `memref.load`, not `vector.load`.** Use + `k.scalarLoad(a.cu_seqlens, ...)` for SMEM operands (typically the + scalar-prefetch path). VMEM args go through the vector-load helpers. + +8. **`pallas_window_params = true` only adds metadata.** It does not change + the kernel body — it just appends `iteration_bounds`, `window_params`, + and the `transform_N` stub funcs to the `func.func` so downstream passes + that consume `window_params` (DMA scheduling, double-buffering) work. + For non-trivial windows or non-VMEM operands, attach a `RefSpec.window` + to override the auto-`synchronous` default. + +9. **`finish` verifies before serializing.** A verifier failure here almost + always means an operand-type mismatch or a missing block terminator — + `scf.yield` operand count must match the loop's result types, and + `func.return` arity must match the declared result types. + +The removed low-level `tpu.relayout` helper has no replacement: upstream deleted +the unused operation and its vector-layout attribute. `Builder.rotate` retains +its static-amount API and emits `tpu.dynamic_rotate` with an i32 constant. diff --git a/dsl/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig new file mode 100644 index 00000000..cffcd658 --- /dev/null +++ b/dsl/mosaic_tpu/builder.zig @@ -0,0 +1,2248 @@ +//! ZML Mosaic TPU DSL — a Zig builder for Mosaic TPU IR (the `tpu` dialect). +//! +//! Args are memrefs (`!memref`) plus optional scalars, +//! iteration-index, and semaphore args. The dialect must be registered into +//! the MLIR context — see `mlir/dialects/mosaic_tpu`. The entire Mosaic stack is +//! backed by the Mosaic dialect in the pinned XLA repository. + +const std = @import("std"); + +const affine = @import("mlir/dialects/affine"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); +/// `vector`-dialect enums re-exported for convenience. +/// Used by `vectorReduction` / `multiReduction`. +pub const CombiningKind = vector.CombiningKind; +const common = @import("dsl/common/builder"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); +/// `tpu`-dialect enums re-exported for convenience. +pub const MemorySpace = mosaic_tpu.MemorySpace; +pub const ReductionKind = mosaic_tpu.ReductionKind; +pub const ContractPrecision = mosaic_tpu.ContractPrecision; +pub const RoundingMode = mosaic_tpu.RoundingMode; +pub const DimensionSemantics = mosaic_tpu.DimensionSemantics; +pub const CoreType = mosaic_tpu.CoreType; +pub const PipelineMode = mosaic_tpu.PipelineMode; +pub const RevisitMode = mosaic_tpu.RevisitMode; + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +pub const pipeline = @import("pipeline.zig"); + +// ============================================================================= +// Value — a handle to an MLIR value inside the kernel body. +// ============================================================================= + +/// MLIR SSA value with an optional back-pointer to its `Builder` for fluent ops. +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + /// Element type of a vector / memref value, or the value's own type if + /// scalar. + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + /// True when this value is a vector (rank ≥ 1). + pub fn isVector(self: Value) bool { + return self.type_().isA(mlir.VectorType) != null; + } + + /// True when this value is a memref. + pub fn isMemRef(self: Value) bool { + return self.type_().isA(mlir.MemRefType) != null; + } + + /// True when the value is shaped (vector / memref / tensor). + pub fn isShaped(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + /// Rank of this value. Scalars have rank 0. + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + /// `i`-th dim size of a shaped value; traps on scalar. + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + /// Shape as a stack-allocated BoundedArray. Use `.constSlice()` for `[]const i64`. + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + } + return false; + } + + /// True if the element type is an integer type. + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + // -------- fluent arithmetic (int/float auto-dispatch from elem type) -------- + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + // -------- fluent comparisons (int predicates default; float uses cmpf) -------- + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + // -------- casts -------- + + /// Numeric cast preserving shape (vector or scalar). Auto-dispatches on + /// source / destination float-vs-int. + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + /// `tpu.bitcast` — same-bitwidth element-type swap. + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + + /// Absolute value (float→absf, int→absi). + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } + + /// Best-effort fold to an `i64` for `arith.constant` (and `addi`/`muli` of folded + /// operands). Returns `null` for anything else; the caller decides what to do. + /// TODO(raph): check if there is no better way + pub fn asConstantInt(self: Value) ?i64 { + if (self.inner.isA(mlir.OpResult) == null) return null; + + const op = self.inner.owner(); + const op_name = op.name(); + + if (std.mem.eql(u8, op_name, "arith.constant")) { + const attr = op.attributeByName("value") orelse return null; + const iattr = attr.isA(mlir.IntegerAttribute) orelse return null; + return iattr.value(i64); + } + + if (std.mem.eql(u8, op_name, "arith.muli")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt(); + const rv = rhs.asConstantInt(); + if (lv) |l| if (rv) |r| return l *% r; + // One side null, the other known: still folds if the known side is 0. + if (lv) |l| if (l == 0) return 0; + if (rv) |r| if (r == 0) return 0; + return null; + } + + if (std.mem.eql(u8, op_name, "arith.addi")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt() orelse return null; + const rv = rhs.asConstantInt() orelse return null; + return lv +% rv; + } + + return null; + } +}; + +// ============================================================================= +// Arg spec — describes one function parameter. +// ============================================================================= + +/// Per-arg specification for building `func.func`'s signature. +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + /// Plain scalar — typically used for grid sizes / dynamic dims passed + /// in by the runtime. `.{ .scalar = .i32 }`. + scalar: DType, + /// Loop / grid iteration index — `index` type with optional + /// `tpu.dimension_semantics` annotation. + /// `.{ .iter = .{ .semantics = .parallel } }`. + iter: IterOpts, + /// Memref (the bread-and-butter kernel arg). The arg is typed as + /// `memref>`. + /// `.{ .ref = .{ .shape = &.{128, 256}, .dtype = .bf16, .memory_space = .vmem } }`. + ref: RefSpec, + /// `memref` — the + /// scratch-array form Pallas uses for semaphore pools. Pallas + /// classifies it as `.scratch`, so it never appears in + /// `window_params` / `transform_N`. + sem_array: SemArraySpec, + /// `!tpu.semaphore` or `!tpu.dma_semaphore`. + /// `.{ .sem = .regular }` / `.{ .sem = .dma }`. + sem: SemKind, + }; + + pub const IterOpts = struct { + semantics: ?DimensionSemantics = null, + }; + + pub const RefSpec = struct { + shape: []const i64, + dtype: DType, + memory_space: MemorySpace = .vmem, + /// Emit `{sc.persistent}` on this memref argument. Sparse-core Pallas + /// kernels mark persistent HBM operands this way. + persistent: bool = false, + /// Block window. `null` ⇒ trivial window (block == operand shape); + /// VMEM trivial windows auto-get `pipeline_mode = synchronous`. + window: ?WindowSpec = null, + /// Pallas role of this operand. Only `.input`/`.output` get a + /// `window_params` entry and a `transform_N` stub. + /// `.scalar_prefetch` refs become trailing args of every + /// `transform_N` body (Pallas's index_map sees them). + /// `.scratch` refs are not visible to transforms or window_params. + role: Role = .input, + }; + + pub const Role = enum { input, output, scratch, scalar_prefetch }; + + pub const WindowSpec = struct { + /// `null` ⇒ use the operand's full shape (trivial window). + /// Determines `window_bounds` and the `transform_N` arity. + block_shape: ?[]const i64 = null, + /// `#tpu.pipeline_mode<…>`. `null` ⇒ auto-`synchronous` for VMEM trivial windows, omitted otherwise. + pipeline_mode: ?PipelineMode = null, + /// `#tpu.revisit_mode<…>`. `null` ⇒ omit. + revisit_mode: ?RevisitMode = null, + /// `#tpu.element_window<[pad_low], [pad_high]>`. `null` ⇒ omit. + element_window: ?ElementWindowSpec = null, + /// What `transform_N` returns. `null` ⇒ all-zero `block_rank` i32s + /// (matching a trivial index_map). One entry per result; length + /// must match `block_shape.len` (or operand rank if `block_shape` + /// is `null`). + transform_returns: ?[]const TransformReturn = null, + }; + + /// One return slot of a `transform_N` body. Mirrors what + /// `lower_jaxpr_to_transform_func` produces — typically a select of + /// program-ids, scalar-prefetch loads, or constants. We only encode + /// the most common forms; drop to a custom transform body for anything + /// fancier. + pub const TransformReturn = union(enum) { + /// Return `arith.constant 0 : i32`. + zero, + /// Return the Nth program-id (`%argN`). + program_id: usize, + }; + + pub const ElementWindowSpec = struct { + pad_low: []const i64, + pad_high: []const i64, + }; + + pub const SemKind = enum { regular, dma }; + + pub const SemArraySpec = struct { + shape: []const i64, + kind: SemKind = .dma, + }; +}; + +pub const FinishError = error{ InvalidMlir, MlirUnexpected } || std.mem.Allocator.Error || std.Io.Writer.Error; + +// ============================================================================= +// Named-param structs. +// ============================================================================= + +pub const VectorLoadOpts = struct { + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, + /// Optional element-wise mask. Must be a `vector<...xi1>` matching the + /// load result shape. + mask: ?Value = null, +}; + +pub const VectorStoreOpts = struct { + strides: []const i32 = &.{}, + mask: ?Value = null, + /// If true emits `tpu.vector_store ... add = true` (atomic-add semantics + /// where supported). + add: bool = false, +}; + +pub const MatmulOpts = struct { + transpose_lhs: bool = false, + transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `mosaic_tpu.dotDimensionNumbersAttr`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub const ReduceOpts = struct { + /// Reduction dim. Defaults to 0. + dim: i64 = 0, +}; + +pub const SortOpts = struct { + descending: bool = false, +}; + +pub const SemSignalOpts = struct { + device_id: ?Value = null, + core_id: ?Value = null, + subcore_id: ?Value = null, +}; + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?Value = null, + device_id: ?Value = null, + core_id: ?Value = null, + subcore_id: ?Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub const WaitDma2Opts = struct { + device_id: ?Value = null, + core_id: ?Value = null, + strict_ordering: bool = false, +}; + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub const RotateOpts = struct { + amount: i32, + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub const CastOpts = struct { + /// Forwarded to `tpu.fptosi` / `tpu.fptoui` / etc. + rounding: ?RoundingMode = null, + /// If true, perform a same-bitwidth `tpu.bitcast` instead of a numeric cast. + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + /// IEEE NaN-propagation toggle. Default is non-propagating (maxnumf/minnumf). + propagate_nan: bool = false, +}; + +// ============================================================================= +// Builder — the main DSL builder. +// ============================================================================= + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, mosaic_tpu, arith, scf, math, cf, memref, vector, affine }) |dialect| dialect.registerDialect(registry); +} + +/// The main DSL builder. Create with `init`, populate the body with helpers, +/// then call `finish` to get the IR string. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + /// Name of the kernel — used when `declareArgs` creates the `func.func`. + name: []const u8, + /// `func.func` op for this kernel. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + func_op: ?*mlir.Operation, + /// Entry block for the `func.func`. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + entry_block: ?*mlir.Block, + block_stack: common.BlockStack = .{}, + /// Cached `arith.constant 0 : index` Value (and the block it lives + /// in) so repeat no-index loads / stores share an SSA value. See + /// `zeroIndex` for the lookup logic. + zero_index: ?Value = null, + zero_index_block: ?*mlir.Block = null, + /// Snapshot of the kernel's ref-args saved at `declareArgsLowOpts` + /// time. `finish` walks this list to emit `transform_N` stubs when + /// `pallas_window_params=true`. Allocated in `arena`. + ref_args: []RefArgSnapshot = &.{}, + /// SMEM scalar-prefetch ref types — appended to every `transform_N` + /// body's argument list. Empty when no `.scalar_prefetch` refs were + /// declared. + prefetch_args: []const *const mlir.Type = &.{}, + /// Number of grid program-ids (i32) at the head of every `transform_N` + /// body. Equals `Opts.dimension_semantics.len`. + grid_dim_count: usize = 0, + /// Whether to emit Pallas-style `transform_N` stubs at `finish` time. + /// Set by `declareArgsLowOpts` from `Opts.pallas_window_params`. + emit_transform_stubs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Whether generated transform functions carry `{sc.is_transform_indices}`. + sc_transform_indices: bool = false, + + /// Per-ref-arg snapshot for `finish`-time `transform_N` stub emission. + /// `skipped` = ANY/HBM/SEMAPHORE memory space (empty window, no transform). + pub const RefArgSnapshot = struct { + block_rank: i64, + skipped: bool, + role: ArgSpec.Role, + transform_returns: ?[]const ArgSpec.TransformReturn = null, + }; + + /// Snapshot of the SMEM scalar-prefetch ref types (after the program-ids) + /// — repeated as trailing args of every `transform_N` body to mirror + /// Pallas's `lower_jaxpr_to_transform_func`. + pub const PrefetchSnapshot = struct { + /// Pre-built MLIR memref types (cached pointers from declareArgs time). + types: []const *const mlir.Type, + }; + + /// Builder-level options. `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + /// `tpu.core_type` on the `func.func`. `null` skips the attr. + core_type: ?CoreType = .tc, + /// One entry per grid dim. Empty list emits an empty array attr + /// (matching Pallas's `dimension_semantics = []`). + dimension_semantics: []const DimensionSemantics = &.{}, + /// Number of leading scalar-prefetch operands. + scalar_prefetch: i64 = 0, + /// Number of trailing scratch refs (memrefs holding kernel-local + /// scratch state). + scratch_operands: i64 = 0, + /// Pallas's `iteration_bounds = array` — the grid shape + /// (one entry per `dimension_semantics`). `null` omits the attr. + iteration_bounds: ?[]const i64 = null, + /// Emit `scalar_prefetch = N : i64`. Sparse-core kernels with no + /// scalar prefetches omit the zero-valued attr. + emit_scalar_prefetch_attr: bool = true, + /// When `true`, auto-emit `window_params = [...]` and trailing + /// `transform_N` stub funcs at `finish` time. Per-arg behavior + /// matches `lowering.py:1014–1131`: ANY/HBM/SEMAPHORE refs get + /// an empty `{}` and no transform; VMEM trivial windows + /// auto-force `pipeline_mode = synchronous`. Override per-arg + /// via `RefSpec.window`. + pallas_window_params: bool = false, + /// Include HBM refs in Pallas-style `window_params`. Tensor-core + /// Pallas lowering skips HBM; sparse-core lowering includes it. + window_hbm_refs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Add `{sc.is_transform_indices}` to generated transform stubs. + sc_transform_indices: bool = false, + }; + + /// Create a sub-module containing a single public `func.func` of the + /// given name, signature, and result types. + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + ) !Builder { + return initOpts(allocator, ctx, name, args, result_types, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, result_types, opts); + return b; + } + + /// Create an empty `Builder` with no `func.func` yet. + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + /// Declare the kernel's args from a named-field struct literal and + /// return a `NamedArgs(Spec)`. + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, &.{}, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + result_types: []const *const mlir.Type, + opts: Opts, + ) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .scalar => .{ .scalar = inner }, + .iter => .{ .iter = .{ + .semantics = if (@hasField(@TypeOf(inner), "semantics")) inner.semantics else null, + } }, + .ref => .{ .ref = .{ + .shape = inner.shape, + .dtype = inner.dtype, + .memory_space = if (@hasField(@TypeOf(inner), "memory_space")) inner.memory_space else .vmem, + .persistent = if (@hasField(@TypeOf(inner), "persistent")) inner.persistent else false, + .window = if (@hasField(@TypeOf(inner), "window")) inner.window else null, + .role = if (@hasField(@TypeOf(inner), "role")) inner.role else .input, + } }, + .sem_array => .{ .sem_array = .{ + .shape = inner.shape, + .kind = if (@hasField(@TypeOf(inner), "kind")) inner.kind else .dma, + } }, + .sem => .{ .sem = inner }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, result_types, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (args, 0..) |a, i| { + arg_types[i] = switch (a.kind) { + .scalar => |dt| dt.toMlir(ctx), + .iter => .index(ctx), + .ref => |r| .memRef( + r.dtype.toMlir(ctx), + r.shape, + null, + r.memory_space.attribute(ctx), + ), + .sem => |s| switch (s) { + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), + }, + .sem_array => |sa| .memRef( + switch (sa.kind) { + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), + }, + sa.shape, + null, + MemorySpace.semaphore_mem.attribute(ctx), + ), + }; + arg_locs[i] = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + // Build per-arg attribute dicts. + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (args, 0..) |a, i| { + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + switch (a.kind) { + .iter => |io| { + if (io.semantics) |sem| { + arg_attrs[i] = .dict(ctx, &.{ + mosaic_tpu.dimensionSemanticsNamedAttr(ctx, sem), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + .ref => |r| { + if (r.persistent) { + arg_attrs[i] = .dict(ctx, &.{ + .named(ctx, "sc.persistent", .unit(ctx)), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + else => arg_attrs[i] = empty_dict, + } + } + + // Kernel-level func.func attributes. MLIR sorts attrs alphabetically on print. + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 12) = .empty; + const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); + for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); + extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", .array(ctx, dim_sem_buf))); + if (opts.iteration_bounds) |bounds| { + extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", .denseArray(ctx, .i64, bounds))); + } + if (opts.emit_scalar_prefetch_attr) { + extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); + } + extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); + if (opts.core_type) |ct| { + extra_attrs.appendAssumeCapacity(mosaic_tpu.coreTypeNamedAttr(ctx, ct)); + } + + // Snapshot the ref args for `finish`-time stub emission. Only + // `.input` / `.output` refs participate in `window_params` and + // `transform_N` — `.scalar_prefetch` and `.scratch` are ignored + // (they're tracked separately for transform-arg expansion). + const arena_alloc = self.arena.allocator(); + var ref_args_buf: std.ArrayList(RefArgSnapshot) = .empty; + var prefetch_types_buf: std.ArrayList(*const mlir.Type) = .empty; + for (args, 0..) |a, i| switch (a.kind) { + .ref => |r| { + if (r.role == .input or r.role == .output) { + const block_shape = if (r.window) |w| + (w.block_shape orelse r.shape) + else + r.shape; + const skipped = isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm); + try ref_args_buf.append(arena_alloc, .{ + .block_rank = @intCast(block_shape.len), + .skipped = skipped, + .role = r.role, + .transform_returns = if (r.window) |w| w.transform_returns else null, + }); + } else if (r.role == .scalar_prefetch) { + try prefetch_types_buf.append(arena_alloc, arg_types[i]); + } + }, + else => {}, + }; + self.ref_args = try arena_alloc.dupe(RefArgSnapshot, ref_args_buf.items); + self.prefetch_args = try arena_alloc.dupe(*const mlir.Type, prefetch_types_buf.items); + self.grid_dim_count = opts.dimension_semantics.len; + self.emit_transform_stubs = opts.pallas_window_params; + self.transform_name_prefix = opts.transform_name_prefix; + self.sc_transform_indices = opts.sc_transform_indices; + + // window_params: one entry per `.input` / `.output` ref. + // ANY/HBM/SEMAPHORE → empty dict, no transform_N. + // Otherwise → {transform_indices, window_bounds, [pipeline_mode], [revisit_mode], [window_kind]}. + // VMEM trivial-window → auto-force pipeline_mode = synchronous. + if (opts.pallas_window_params) { + var idx: usize = 0; + const wp_buf = try scratch.alloc(*const mlir.Attribute, self.ref_args.len); + for (args) |a| switch (a.kind) { + .ref => |r| { + if (r.role != .input and r.role != .output) continue; + if (isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm)) { + wp_buf[idx] = .dict(ctx, &.{}); + idx += 1; + continue; + } + + const w = r.window orelse ArgSpec.WindowSpec{}; + const block_shape = w.block_shape orelse r.shape; + // `bm.has_trivial_window()` in pallas — block_shape == + // operand_shape AND the index_map is the identity + // (returns all zeros). + const has_trivial_index_map = if (w.transform_returns) |trs| blk: { + for (trs) |tr| { + if (tr != .zero) break :blk false; + } + break :blk true; + } else true; + const trivial = std.mem.eql(i64, block_shape, r.shape) and has_trivial_index_map; + + const resolved_pipeline: ?PipelineMode = + w.pipeline_mode orelse blk: { + // VMEM + trivial window ⇒ Buffered(1) ⇒ synchronous. + if (r.memory_space == .vmem and trivial) break :blk .synchronous; + break :blk null; + }; + + const sym_name = if (opts.transform_name_prefix) |prefix| + try std.fmt.allocPrint(scratch, "{s}_transform_{d}", .{ prefix, idx }) + else + try std.fmt.allocPrint(scratch, "transform_{d}", .{idx}); + + var inner: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (resolved_pipeline) |pm| { + inner.appendAssumeCapacity(.named(ctx, "pipeline_mode", pm.attribute(ctx))); + } + if (w.revisit_mode) |rm| { + inner.appendAssumeCapacity(.named(ctx, "revisit_mode", rm.attribute(ctx))); + } + inner.appendAssumeCapacity(.named(ctx, "transform_indices", .flatSymbolRef(ctx, sym_name))); + inner.appendAssumeCapacity(.named(ctx, "window_bounds", .denseArray(ctx, .i64, block_shape))); + if (w.element_window) |ew| { + inner.appendAssumeCapacity(.named(ctx, "window_kind", mosaic_tpu.elementWindowAttr(ctx, ew.pad_low, ew.pad_high))); + } + wp_buf[idx] = .dict(ctx, inner.constSlice()); + idx += 1; + }, + else => {}, + }; + extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", .array(ctx, wp_buf))); + } + + const func_op = func.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = result_types, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + // Body is empty at declare time — verify only at `finish`. + .verify = false, + .extra_attributes = extra_attrs.constSlice(), + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// The i-th entry block argument. + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + /// Unpack a slice of `Value`s into an arena-allocated slice of raw MLIR + /// value pointers. + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + /// Emit `op` into the current block and return its first result as a Value. + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + // ==================== type helpers ==================== + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn vectorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return .vector(shape, dtype.toMlir(self.ctx)); + } + + pub fn memRefTy(self: *const Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) *const mlir.Type { + return .memRef(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + /// Replace `src`'s element type with `out_dtype`, preserving rank/shape. + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isVector()) return .vector(src.shape().constSlice(), out_elem); + if (src.isMemRef()) { + const mr = src.type_().isA(mlir.MemRefType).?; + return .memRef(out_elem, src.shape().constSlice(), null, mr.memorySpace()); + } + return out_elem; + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f16 => self.emit(arith.constant_float(self.ctx, value, .f16, self.loc())), + .bf16 => self.emit(arith.constant_float(self.ctx, value, .bf16, self.loc())), + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + // fp8 has no direct arith.constant; emit f32 then truncf. + else => self.arithTruncf(self.emitFloat(.f32, value), dtype), + }; + } + + /// Constant `index`-typed value (for memref / loop bounds). + pub fn constIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + /// `openFor`'s default lower bound, an `index` constant. + fn indexBound(self: *Builder, value: anytype) Value { + return self.constIndex(intBits(value, 64, "index")); + } + + /// Lift a Zig scalar (or pass-through Value) to a DSL constant. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); + } + + /// Build an arith.constant of the given MLIR element type from a Zig + /// numeric. Used for matching the type of an existing reference value. + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + if (elem.eql(.index(self.ctx))) return self.constIndex(intBits(value, 64, "index")); + return self.constant(self.mlirElemToDType(elem), value); + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Splat a scalar to a vector. Zig scalars take the dense-splat fast path + /// (`arith.constant dense`); a `Value` falls back to `vector.broadcast`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64, dtype: DType) Value { + const T = @TypeOf(value); + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); + if (T == Value) return self.emit(vector.broadcast(self.ctx, value.inner, ty, self.loc())); + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_f(self.ctx, v_f64, ty, self.loc())); + } + const v_i64: i64 = switch (@typeInfo(T)) { + .comptime_int => @intCast(value), + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_i(self.ctx, v_i64, ty, self.loc())); + } + + /// Zero-valued vector of `shape x dtype`. + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 0), shape, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 1), shape, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(value, shape, dtype); + } + + /// Lift scalars and broadcast scalar → vector to match the other operand. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + // integer-width promotion (int family only). + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + // scalar↔vector splat. + if (av.isVector() and !bv.isVector()) { + const dt = self.mlirElemToDType(av.elemType()); + bv = self.splat(bv, av.shape().constSlice(), dt); + } + if (!av.isVector() and bv.isVector()) { + const dt = self.mlirElemToDType(bv.elemType()); + av = self.splat(av, bv.shape().constSlice(), dt); + } + + return .{ av, bv }; + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + /// Mirrors `jax.numpy.floor_divide` (`jax/_src/numpy/ufuncs.py:2487`): + /// truncating `divsi` plus the sign-correcting expansion + /// `q - 1 if (sign(a) != sign(b) and rem != 0) else q`. + /// `lhs` must be a vector of i32. `rhs` is `anytype` — passing a + /// scalar comptime int produces the dense<...> splat Pallas emits. + /// JAX's MLIR canonicalizer does not fold the expansion away even + /// when one operand is a constant, so this helper is needed for + /// IR-equivalent floor-div with `pallas_call`. + pub fn divFloor(self: *Builder, lhs: Value, rhs: anytype) Value { + const shape = lhs.shape().constSlice(); + const rhs_v: Value = if (@TypeOf(rhs) == Value) rhs else self.splat(rhs, shape, .i32); + const zero_v = self.zeros(shape, .i32); + const one_v = self.ones(shape, .i32); + const q = self.divsi(lhs, rhs_v); + const sign_a = self.subi( + self.extui(self.cmpi(.sgt, lhs, zero_v), .i32), + self.extui(self.cmpi(.slt, lhs, zero_v), .i32), + ); + const sign_b = self.subi( + self.extui(self.cmpi(.sgt, rhs_v, zero_v), .i32), + self.extui(self.cmpi(.slt, rhs_v, zero_v), .i32), + ); + const sign_diff = self.cmpi(.ne, sign_a, sign_b); + const rem_nz = self.cmpi(.ne, self.remsi(lhs, rhs_v), zero_v); + const cond = self.andi(sign_diff, rem_nz); + return self.select(cond, self.subi(q, one_v), q); + } + + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + const xv, const yv = self.coerce(x, y); + const c = if (condition.isVector() == xv.isVector()) condition else blk: { + // Splat a scalar predicate to match a vector x/y. + if (xv.isVector()) { + break :blk self.splat(condition, xv.shape().constSlice(), .i1); + } + break :blk condition; + }; + return self.select(c, xv, yv); + } + + /// Elementwise max with auto-broadcast scalar→vector. Use `maxOpts` + /// for NaN-propagation control. + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.maximumf(l, r) else self.maxnumf(l, r); + } + return self.maxsi(l, r); + } + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.minimumf(l, r) else self.minnumf(l, r); + } + return self.minsi(l, r); + } + + // ==================== arith casts ==================== + + pub fn extsi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithExtf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithTruncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithSitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithUitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// `arith.index_cast` — cast between integer and `index` types. + pub fn indexCast(self: *Builder, src: Value, target: *const mlir.Type) Value { + return self.emit(arith.index_cast(self.ctx, src.inner, target, self.loc())); + } + + /// Cast a scalar integer (i32 / i64 / …) to the `index` type. Common + /// pattern for using i32 program-ids and scalar-prefetch values as + /// memref offsets. + pub fn toIndex(self: *Builder, src: Value) Value { + return self.indexCast(src, .index(self.ctx)); + } + + // ==================== TPU casts ==================== + + /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. + pub fn bitCast(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// Numeric cast with auto-dispatch. Routes through `arith.*` ops for the + /// common float / int conversions (which don't carry a rounding-mode + /// attribute). The `tpu.*` cast variants (`tpuTruncf`, `tpuFptosi`, …) + /// are exposed separately for callers that need rounding-mode control — + /// pass `opts.rounding` to opt in. + pub fn cast(self: *Builder, src: Value, dt: DType) Value { + return self.castOpts(src, dt, .{}); + } + pub fn castOpts(self: *Builder, src: Value, dt: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitCast(src, dt); + const cur_elem = src.elemType(); + const tgt_elem = dt.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dt); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.arithExtf(src, dt); + return self.arithTruncf(src, dt); + } + if (cur_is_float and !tgt_is_float) return self.arithFptosi(src, dt); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.arithUitofp(src, dt); + return self.arithSitofp(src, dt); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.extsi(src, dt); + if (tgt_bw < cur_bw) return self.trunci(src, dt); + return self.arithBitcast(src, dt); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + /// `exp2(x)` → `exp(x * ln(2))`. Matches Pallas's TPU lowering rule + /// (`jax/_src/pallas/mosaic/lowering.py:_exp2_lowering_rule`), which + /// expands `lax.exp2_p` into the same `x * ln2` pattern in forward-compat + /// mode. Materializes as an `arith.constant` (matching x's element type) + /// + `arith.mulf x, ln2` + `math.exp`. For the raw `math.exp2` op + /// (post-2025-07-26 cloud TPUs), use `mathExp2`. + pub fn exp2(self: *Builder, x: Value) Value { + const ln2 = self.constant(self.mlirElemToDType(x.elemType()), @as(f64, 0.6931471805599453)); + const ln2_bc = if (x.isShaped()) self.broadcastTo(ln2, x.shape().constSlice()) else ln2; + return self.exp(self.mulf(x, ln2_bc)); + } + pub fn mathExp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log1p(self: *Builder, x: Value) Value { + return self.emit(math.log1p(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + + // ==================== TPU memory: vector_load / vector_store / scalar load/store ==================== + + /// Build the result vector type for a full-shape `tpu.vector_load` from + /// the input ref's shape + element type. + fn refVectorType(self: *Builder, ref: Value) *const mlir.Type { + _ = self; + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("Builder.refVectorType: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + return .vector(s.constSlice(), mr.elementType()); + } + + /// Full-shape `vector.load`. Use `tpuVectorLoad` for sublane stride / masking. + pub fn refLoad(self: *Builder, ref: Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, &.{}); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `vector.load(ref, indices)` — explicit `index`-typed offsets. + pub fn vectorLoadAt(self: *Builder, ref: Value, indices: []const Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `tpu.vector_load` — sublane stride + optional element-wise mask. + pub fn tpuVectorLoad(self: *Builder, ref: Value, indices: []const Value, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); + } + + /// `tpu.vector_load` with an explicit result vector shape. + pub fn tpuVectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = mlir.Type.vector(shape, ref.elemType()); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); + } + + /// `tpu.vector_store(value, ref)` — full-shape vector store. + pub fn refStore(self: *Builder, ref: Value, value: Value) void { + self.refStoreOpts(ref, value, &.{}, .{}); + } + + pub fn vectorStoreAt(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + self.refStoreOpts(ref, value, indices, .{}); + } + + pub fn refStoreOpts(self: *Builder, ref: Value, value: Value, indices: []const Value, opts: VectorStoreOpts) void { + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + _ = mosaic_tpu.vector_store(self.ctx, value.inner, ref.inner, idx_inner, .{ .strides = opts.strides, .add = opts.add, .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); + } + + /// `memref.load` — scalar load. Indices must be `index`-typed. + pub fn scalarLoad(self: *Builder, ref: Value, indices: []const Value) Value { + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(memref.load(self.ctx, ref.inner, idx_inner, .{}, self.loc())); + } + + /// `memref.store` — scalar store. + pub fn scalarStore(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + const idx_inner = self.indicesOrZero(ref, indices); + _ = memref.store(self.ctx, value.inner, ref.inner, idx_inner, .{}, self.loc()).appendTo(self.currentBlock()); + } + + /// Returns a `*const mlir.Value` slice of length `ref.rank()`, padding with + /// a cached `arith.constant 0 : index` when `indices` is empty. + fn indicesOrZero(self: *Builder, ref: Value, indices: []const Value) []const *const mlir.Value { + const r = ref.rank(); + if (indices.len == r) return self.innerSlice(indices); + std.debug.assert(indices.len == 0); + const zero = self.zeroIndex(); + const out = self.arena.allocator().alloc(*const mlir.Value, r) catch @panic("Builder.indicesOrZero OOM"); + for (out) |*p| p.* = zero.inner; + return out; + } + + /// Return a cached `arith.constant 0 : index` Value, lazily emitting + /// it the first time it's needed. The cache is invalidated whenever + /// the insertion block changes — `index` constants are scoped to the + /// block they're emitted in. + fn zeroIndex(self: *Builder) Value { + const block = self.currentBlock(); + if (self.zero_index) |z| { + if (self.zero_index_block == block) return z; + } + const z = self.constIndex(0); + self.zero_index = z; + self.zero_index_block = block; + return z; + } + + /// `tpu.strided_load` — multi-dim strided load. Result shape is derived + /// from `ref.shape / strides` (per-dim ceil divide). + pub fn stridedLoad(self: *Builder, ref: Value, indices: []const Value, strides: []const i32) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoad: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| { + const stride: i64 = if (i < strides.len) strides[i] else 1; + const dim = mr.dimension(i); + s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); + } + const result_ty = mlir.Type.vector(s.constSlice(), mr.elementType()); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_load` with an explicit result shape — when you know the + /// load shape doesn't match the simple `ref.shape / strides` form (e.g. + /// inner-dim crop). + pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); + const result_ty = mlir.Type.vector(result_shape, mr.elementType()); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_store` — multi-dim strided store. + pub fn stridedStore(self: *Builder, ref: Value, value: Value, indices: []const Value, strides: []const i32) void { + _ = mosaic_tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== TPU compute: matmul / iota / reciprocal ==================== + + /// `tpu.matmul(lhs, rhs, acc)`. Result type is taken from `acc`. Use + /// `matmulOpts` to set transposes / precision / dimension_numbers. + pub fn matmul(self: *Builder, lhs: Value, rhs: Value, acc: Value) Value { + return self.matmulOpts(lhs, rhs, acc, .{}); + } + + pub fn matmulOpts(self: *Builder, lhs: Value, rhs: Value, acc: Value, opts: MatmulOpts) Value { + return self.emit(mosaic_tpu.matmul( + self.ctx, + lhs.inner, + rhs.inner, + acc.inner, + acc.type_(), + .{ + .transpose_lhs = opts.transpose_lhs, + .transpose_rhs = opts.transpose_rhs, + .transpose_lhs_hint = opts.transpose_lhs_hint, + .precision = opts.precision, + .dimension_numbers = opts.dimension_numbers, + }, + self.loc(), + )); + } + + /// `tpu.iota` over the given shape / dimensions. Pass `null` for + /// `dimensions` to use the default. + pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); + return self.emit(mosaic_tpu.iota(self.ctx, ty, .{ .dimensions = dimensions }, self.loc())); + } + + /// 1-D `arange(0, n)`. TPU layout pass requires rank ≥ 2 with the lane dim + /// trailing, so this emits a `<1xN>` `tpu.iota` then `shape_cast` to ``. + pub fn arange(self: *Builder, n: i64, dtype: DType) Value { + const elem = dtype.toMlir(self.ctx); + const ty_2d = .vector(&.{ 1, n }, elem); + const iota_2d = self.emit(mosaic_tpu.iota(self.ctx, ty_2d, .{ .dimensions = &.{1} }, self.loc())); + return self.shapeCast(iota_2d, &.{n}); + } + + /// `tpu.reciprocal(x)` with default options. + pub fn reciprocal(self: *Builder, x: Value) Value { + return self.reciprocalOpts(x, .{}); + } + + pub fn reciprocalOpts(self: *Builder, x: Value, opts: ReciprocalOpts) Value { + return self.emit(mosaic_tpu.reciprocal(self.ctx, x.inner, .{ + .approx = opts.approx, + .full_range = opts.full_range, + }, self.loc())); + } + + // ==================== Vector dialect helpers ==================== + // + // The `tpu.*` shape ops (`reshape`, `transpose`, + // `broadcastInSublanes`) are Mosaic-specific layout-aware variants. + + /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + const ty = mlir.Type.vector(shape, src.elemType()); + return self.emit(vector.broadcast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.shape_cast` — reshape preserving total lanes. `reshape` emits `tpu.reshape` instead. + pub fn shapeCast(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.Type.vector(new_shape, src.elemType()); + return self.emit(vector.shape_cast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.extract` — extract a scalar (or lower-rank vector) at a + /// static position. Result shape = `src.shape[position.len..]`. + pub fn vectorExtract(self: *Builder, src: Value, position: []const i64) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (position.len..sh.len) |i| out.appendAssumeCapacity(sh.get(i)); + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + .vector(out.constSlice(), src.elemType()); + return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); + } + + /// `vector.extract_strided_slice` — statically slice a vector. + pub fn vectorExtractStridedSlice(self: *Builder, src: Value, offsets: []const i64, sizes: []const i64, strides: []const i64) Value { + const ty = mlir.Type.vector(sizes, src.elemType()); + return self.emit(vector.extract_strided_slice(self.ctx, src.inner, .{ + .offsets = offsets, + .sizes = sizes, + .strides = strides, + }, ty, self.loc())); + } + + /// `vector.reduction ` — reduce a 1-D vector to a scalar. + /// Emits the bare upstream op; on real TPU the layout pass may reject + /// 1-D vectors, so prefer `reduceToScalar` for kernel-level reductions. + pub fn vectorReductionFlat(self: *Builder, kind: CombiningKind, src: Value) Value { + return self.emit(vector.reduction(self.ctx, kind, src.inner, src.elemType(), .{}, self.loc())); + } + + /// 1-D vector → scalar. TPU vectors are 2-D, so this adds a leading + /// singleton before reducing: shape_cast→<1xN>, multi_reduction[1]→<1>, extract[0]. + /// `acc` is the reduction identity (e.g. `k.zeros(&.{1}, .f32)` for sum). + pub fn reduceToScalar(self: *Builder, kind: CombiningKind, src: Value, acc: Value) Value { + std.debug.assert(src.rank() == 1); + const n = src.shape().get(0); + const src_2d = self.shapeCast(src, &.{ 1, n }); + const reduced = self.multiReduction(kind, src_2d, acc, &.{1}); + return self.vectorExtract(reduced, &.{0}); + } + + /// `vector.multi_reduction ` — reduce a multi-D vector along + /// the given axes with an explicit accumulator. Result rank = + /// `src.rank - axes.len`. + pub fn multiReduction( + self: *Builder, + kind: CombiningKind, + src: Value, + acc: Value, + axes: []const i64, + ) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + outer: for (0..sh.len) |i| { + for (axes) |a| if (a == @as(i64, @intCast(i))) continue :outer; + out.appendAssumeCapacity(sh.get(i)); + } + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + .vector(out.constSlice(), src.elemType()); + return self.emit(vector.multi_reduction(self.ctx, kind, src.inner, acc.inner, axes, ty, self.loc())); + } + + /// `vector.load` with an explicit result shape — for `ref[i, j, :, k:k+BLOCK]`-style sliced loads. + pub fn vectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64) Value { + const idx_inner = self.indicesOrZero(ref, indices); + const ty = mlir.Type.vector(shape, ref.elemType()); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, ty, .{}, self.loc())); + } + + // ==================== TPU shape ==================== + + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.Type.vector(new_shape, src.elemType()); + return self.emit(mosaic_tpu.reshape(self.ctx, src.inner, ty, self.loc())); + } + + pub fn repeat(self: *Builder, src: Value, dimension: i32, times: i32) Value { + const in_shape = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| { + const d = in_shape.get(i); + out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); + } + if (times == 1) return src; + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); + const sources = self.arena.allocator().alloc(*const mlir.Value, @intCast(times)) catch @panic("OOM"); + @memset(sources, src.inner); + return self.emit(mosaic_tpu.concatenate(self.ctx, sources, dimension, ty, self.loc())); + } + + pub fn concatenate(self: *Builder, sources: []const Value, dimension: i32) Value { + std.debug.assert(sources.len > 0); + const ax: usize = @intCast(dimension); + var sum: i64 = 0; + for (sources) |s| sum += s.dim(ax); + const head = sources[0].shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); + const ty = mlir.Type.vector(out.constSlice(), sources[0].elemType()); + return self.emit(mosaic_tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); + } + + pub fn transpose(self: *Builder, src: Value, permutation: []const i64) Value { + const in_shape = src.shape(); + std.debug.assert(permutation.len == in_shape.len); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); + return self.emit(mosaic_tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); + } + + pub fn broadcastInSublanes(self: *Builder, src: Value, lane: i32) Value { + // Result shape = src shape (the op replicates within sublanes). + return self.emit(mosaic_tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); + } + + pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { + const amount = self.constant(.i32, opts.amount); + return self.emit(mosaic_tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ + .dimension = opts.dimension, + .stride = opts.stride, + .stride_dimension = opts.stride_dimension, + }, self.loc())); + } + + // ==================== TPU reductions / scan / sort ==================== + + /// `tpu.all_reduce(input, dim, kind)` over a single dim. + pub fn allReduce(self: *Builder, input: Value, kind: ReductionKind, dim: i64) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); + const ty = mlir.Type.vector(out.constSlice(), input.elemType()); + return self.emit(mosaic_tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); + } + + /// `tpu.reduce_index(input, kind, axis)` — drop axis, replace with + /// `index`-valued indices (integer). + pub fn reduceIndex(self: *Builder, input: Value, kind: ReductionKind, axis: i32) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (in_shape.constSlice(), 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + const ty = mlir.Type.vector(out.constSlice(), .int(self.ctx, .i32)); + return self.emit(mosaic_tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); + } + + /// Scan the last dimension; a mask must be rank one with that dimension + /// as its length. The result has the same shape as the input. + pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + return self.emit(mosaic_tpu.scan(self.ctx, input.inner, kind, input.type_(), .{ .mask = m_inner }, self.loc())); + } + + /// `tpu.sort(keys, values, mask?)` — returns (output_mask, sorted_keys, sorted_values). + pub fn sort(self: *Builder, keys: Value, values: Value, mask: ?Value, opts: SortOpts) [3]Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + const out_mask_ty = if (mask) |m| m.type_() else blk: { + // Default: vector. + break :blk .vector(keys.shape().constSlice(), .integer(self.ctx, .i1)); + }; + const op = mosaic_tpu.sort( + self.ctx, + keys.inner, + values.inner, + out_mask_ty, + keys.type_(), + values.type_(), + .{ .descending = opts.descending, .mask = m_inner }, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + .{ .inner = op.result(2), .kernel = self }, + }; + } + + // ==================== TPU memref-shape ops ==================== + + pub fn memRefSlice( + self: *Builder, + mem_ref: Value, + base_idx: []const Value, + result_shape: []const i64, + dynamic_sizes: []const Value, + ) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_slice(self.ctx, mem_ref.inner, .{ + .base_idx = self.innerSlice(base_idx), + .dynamic_sizes = self.innerSlice(dynamic_sizes), + }, result_ty, self.loc())); + } + + pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefAlloca(self: *Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) Value { + return self.emit(memref.alloca(self.ctx, self.memRefTy(shape, dtype, memory_space), &.{}, &.{}, .{}, self.loc())); + } + + pub fn memRefBitcast(self: *Builder, mem_ref: Value, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + const result_ty = .memRef(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + /// `tpu.memref_bitcast` with an explicit result shape — needed for the + /// `bf16 → i32` packing trick in ragged-paged-attention's + /// `strided_load_kv` path (`<64x128xbf16>` → `<32x128xi32>`). + pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn assumeLayout(self: *Builder, src: Value) Value { + return self.emit(mosaic_tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); + } + + pub fn eraseMemRefLayout(self: *Builder, mem_ref: Value) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("eraseMemRefLayout: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + // Result drops the layout but keeps shape, element type, memory space. + const result_ty = .memRef(mr.elementType(), s.constSlice(), null, mr.memorySpace()); + return self.emit(mosaic_tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + // ==================== Mask building ==================== + + /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. + pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { + const ty = mlir.Type.vector(shape, .integer(self.ctx, .i1)); + return self.emit(mosaic_tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); + } + + /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two + /// `index` constants and a `tpu.create_mask`. + pub fn rangeMask(self: *Builder, limits: []const Value, shape: []const i64) Value { + const zero = self.constIndex(0); + const lows = self.arena.allocator().alloc(Value, limits.len) catch @panic("rangeMask OOM"); + for (lows) |*l| l.* = zero; + return self.createMask(lows, limits, shape); + } + + // ==================== Semaphores / DMA / barriers ==================== + + pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { + const elem = switch (kind) { + .regular => mosaic_tpu.semaphoreType(self.ctx), + .dma => mosaic_tpu.dmaSemaphoreType(self.ctx), + }; + const ty = mlir.Type.memRef(elem, &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); + return self.emit(mosaic_tpu.sem_alloc(self.ctx, ty, self.loc())); + } + + pub fn semBarrier(self: *Builder) Value { + const ty = mlir.Type.memRef(mosaic_tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); + return self.emit(mosaic_tpu.sem_barrier(self.ctx, ty, self.loc())); + } + + pub fn semRead(self: *Builder, semaphore: Value) Value { + return self.emit(mosaic_tpu.sem_read(self.ctx, semaphore.inner, self.loc())); + } + + pub fn semWait(self: *Builder, semaphore: Value, amount: anytype) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + _ = mosaic_tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn semSignal(self: *Builder, semaphore: Value, amount: anytype) void { + self.semSignalOpts(semaphore, amount, .{}); + } + + pub fn semSignalOpts(self: *Builder, semaphore: Value, amount: anytype, opts: SemSignalOpts) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; + _ = mosaic_tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ + .device_id = dev, + .core_id = core, + .subcore_id = subcore, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrier(self: *Builder, barrier_id: Value) void { + _ = mosaic_tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn enqueueDma(self: *Builder, source: Value, target: Value, target_semaphore: Value, opts: EnqueueDmaOpts) void { + const src_sem: ?*const mlir.Value = if (opts.source_semaphore) |s| s.inner else null; + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; + _ = mosaic_tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ + .source_semaphore = src_sem, + .device_id = dev, + .core_id = core, + .subcore_id = subcore, + .priority = opts.priority, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn waitDma2(self: *Builder, semaphore: Value, src: Value, dst: Value, opts: WaitDma2Opts) void { + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + _ = mosaic_tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ + .device_id = dev, + .core_id = core, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn deviceId(self: *Builder) Value { + return self.emit(mosaic_tpu.device_id(self.ctx, self.loc())); + } + + /// `tpu.trace_start` — pair with `traceStop`. Used by Pallas to mark + /// einsum boundaries (e.g. `"nd,md->nm"`). + pub fn traceStart(self: *Builder, level: i32, message: []const u8) void { + _ = mosaic_tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); + } + + pub fn traceStop(self: *Builder) void { + _ = mosaic_tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); + } + + /// Build a `#tpu.dot_dimension_numbers<...>` attribute for `matmulOpts`. + pub fn dotDimensionNumbers( + self: *Builder, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, + ) *const mlir.Attribute { + return mosaic_tpu.dotDimensionNumbersAttr( + self.ctx, + lhs_contracting, + rhs_contracting, + lhs_non_contracting, + rhs_non_contracting, + output_dim_order, + lhs_batch, + rhs_batch, + ); + } + + pub fn delay(self: *Builder, cycles: anytype) void { + const c_ = if (@TypeOf(cycles) == Value) cycles else self.lift(cycles); + _ = mosaic_tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn assumeMultiple(self: *Builder, src: Value, multiple: i32) Value { + return self.emit(mosaic_tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); + } + + pub fn openRegion(self: *Builder) *mlir.Block { + const block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(block); + return block; + } + + pub fn closeRegion(self: *Builder, block: *mlir.Block) void { + _ = mosaic_tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); + self.popBlock(); + _ = mosaic_tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== SCF regions ==================== + + /// Open a scoped `scf.for`. Literal bounds take the type of a Value bound, + /// else they are `index`. `inits` is `.{v1, v2, ...}` or an array. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, indexBound), inits, opts); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== finalization ==================== + + /// Append `func.return` with the given results, verify the module, and + /// serialize to a NUL-terminated MLIR string owned by the kernel's + /// allocator. Caller must `defer allocator.free(ir)`. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + return self.finishOpts(results, .{}); + } + + pub const FinishOpts = struct { + /// Run `canonicalize,cse` on the module before serializing — lets + /// the user code stay one-line-per-Pallas-line and lean on MLIR + /// to dedupe duplicated constants / identical subexpressions, the + /// same way Pallas does (its lowering doesn't dedupe either; the + /// IR you read in `debug=True` is post-canonicalize). + canonicalize: bool = false, + }; + + pub fn finishOpts(self: *Builder, results: []const Value, opts: FinishOpts) FinishError![:0]const u8 { + const current = self.currentBlock(); + _ = func.returns(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); + + if (self.emit_transform_stubs) self.appendTransformStubs(); + + try common.verify(self.module, "dsl/mosaic_tpu"); + + if (opts.canonicalize) { + const pm = mlir.PassManager.init(self.ctx); + defer pm.deinit(); + // `canonicalize` folds `x * 1`, `x + 0`, etc.; `cse` then + // dedupes identical constants and subexpressions. Same + // pipeline `zml/Compiler.zig` runs on every compiled module. + const opm = pm.asOpPassManager(); + inline for (.{ "canonicalize", "cse", "canonicalize" }) |pass| { + try opm.addPipeline(pass); + } + try pm.runOnOp(self.module.operation()); + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = false, + })}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } + + /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. + /// Each takes `(i32 ×grid_dim_count, prefetch_refs...)` and returns + /// `block_rank` i32 values per `RefArgSnapshot.transform_returns` + /// (default: all zeros, matching a trivial index_map). + fn appendTransformStubs(self: *Builder) void { + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const i32_ty: *const mlir.Type = .int(ctx, .i32); + + const arena_alloc = self.arena.allocator(); + const total_args = self.grid_dim_count + self.prefetch_args.len; + const arg_types = arena_alloc.alloc(*const mlir.Type, total_args) catch @panic("OOM"); + const arg_locs = arena_alloc.alloc(*const mlir.Location, total_args) catch @panic("OOM"); + for (0..self.grid_dim_count) |i| { + arg_types[i] = i32_ty; + arg_locs[i] = unknown_loc; + } + for (self.prefetch_args, 0..) |ty, i| { + arg_types[self.grid_dim_count + i] = ty; + arg_locs[self.grid_dim_count + i] = unknown_loc; + } + + for (self.ref_args, 0..) |ra, idx| { + if (ra.skipped) continue; + + const rank: usize = @intCast(ra.block_rank); + const result_types = arena_alloc.alloc(*const mlir.Type, rank) catch @panic("OOM"); + for (result_types) |*r| r.* = i32_ty; + + const block = mlir.Block.init(arg_types, arg_locs); + + const c0 = arith.constant_int(ctx, 0, i32_ty, unknown_loc); + _ = c0.appendTo(block); + + const ret_vals = arena_alloc.alloc(*const mlir.Value, rank) catch @panic("OOM"); + const returns = ra.transform_returns; + for (ret_vals, 0..) |*v, i| { + if (returns) |r_list| { + if (i < r_list.len) { + v.* = switch (r_list[i]) { + .zero => c0.result(0), + .program_id => |pid| block.argument(pid), + }; + continue; + } + } + v.* = c0.result(0); + } + _ = func.returns(ctx, ret_vals, unknown_loc).appendTo(block); + + const name = if (self.transform_name_prefix) |prefix| + std.fmt.allocPrint(arena_alloc, "{s}_transform_{d}", .{ prefix, idx }) catch @panic("OOM") + else blk: { + var name_buf: [32]u8 = undefined; + break :blk arena_alloc.dupe(u8, std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable) catch @panic("OOM"); + }; + + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (self.sc_transform_indices) { + extra_attrs.appendAssumeCapacity(.named(ctx, "sc.is_transform_indices", .unit(ctx))); + } + + const stub_op = func.func(ctx, .{ + .name = name, + .args = arg_types, + .results = result_types, + .block = block, + .location = unknown_loc, + .verify = false, + .extra_attributes = extra_attrs.constSlice(), + }); + _ = stub_op.appendTo(self.module.body()); + } + } +}; + +/// Memory spaces that don't support windowing (empty window_params, no transform_N). +fn isSkippedWindowSpace(ms: MemorySpace) bool { + return switch (ms) { + .hbm, .any, .semaphore_mem => true, + else => false, + }; +} + +// ==================== scope-type aliases pinned to Mosaic's Builder ==================== + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "static rotate uses the current dynamic rotate operation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var b = try Builder.init(std.testing.allocator, ctx, "rotate", &.{}, &.{}); + defer b.deinit(); + const input = b.zeros(&.{ 8, 128 }, .f32); + const rotated = b.rotate(input, .{ .amount = 3, .dimension = 1 }); + try std.testing.expectEqualStrings("tpu.dynamic_rotate", rotated.inner.owner().name()); + const amount: Value = .{ .inner = rotated.inner.owner().operand(1), .kernel = &b }; + try std.testing.expectEqual(@as(?i64, 3), amount.asConstantInt()); + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); +} diff --git a/dsl/mosaic_tpu/dtype.zig b/dsl/mosaic_tpu/dtype.zig new file mode 100644 index 00000000..549c69da --- /dev/null +++ b/dsl/mosaic_tpu/dtype.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Mosaic-TPU DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig new file mode 100644 index 00000000..01264373 --- /dev/null +++ b/dsl/mosaic_tpu/pipeline.zig @@ -0,0 +1,1147 @@ +//! Zig port of `pltpu.emit_pipeline` (jax/_src/pallas/mosaic/pipeline.py) — +//! emits lowered `tpu`-dialect IR byte-equivalent to `pallas_call`. +//! +//! `.blocked`/`.squeezed`/one `.bounded_slice` block dim; trivial-windowing +//! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. + +const std = @import("std"); +const func = @import("mlir/dialects/func"); +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); +const builder = @import("builder.zig"); + +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); + +const Builder = builder.Builder; +const Value = builder.Value; +const DType = builder.DType; +const MemorySpace = builder.MemorySpace; +const FinishError = builder.FinishError; + +/// `ShapedType::kDynamic` — sentinel for a `?` dim in a memref shape. +const kDyn: i64 = std.math.minInt(i64); + +/// One entry of `pl.BlockSpec.block_shape`. +pub const BlockDim = union(enum) { + /// `pl.Blocked(n)` — windowed dim of size `n`. + blocked: i64, + /// `pl.Squeezed()` / `None` — dim dropped in the window. + squeezed, + /// `pl.BoundedSlice(n)` — index_map returns `.slice`; `n` is the max. + bounded_slice: i64, +}; + +/// One entry returned by `pl.BlockSpec.index_map`. +pub const BlockIndex = union(enum) { + /// `pl.ds(idx*block, block)` — for `.blocked`/`.squeezed` dims. + scalar: Value, + /// `pl.ds(start, size)` — for `.bounded_slice` dims. + slice: struct { start: Value, size: Value }, +}; + +/// `pl.BlockSpec.index_map`. `ctx` is opaque user data (Zig has no closures). +pub const IndexMapFn = *const fn (b: *Builder, indices: []const Value, ctx: ?*anyopaque) []const BlockIndex; + +/// Working window for one pipeline operand. Body must re-emit +/// `memref_slice`/`squeeze` at each use site. `slot == null` ⇒ trivial-windowing; +/// `buf == null` ⇒ no operand. +pub const RefWindow = struct { buf: ?Value = null, slot: ?Value = null }; + +/// The pipeline body — `pallas_call`'s inner kernel. +pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: []const RefWindow, scratches: []const Value, ctx: ?*anyopaque) FinishError!void; + +// Optional overrides for a buffered reference's DMA operations. +// Unspecified operations retain the standard pipeline behavior. +pub const BufferedRefStrategy = struct { + ctx: ?*anyopaque = null, + copy_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool, ctx: ?*anyopaque) void = null, + wait_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + copy_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + wait_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, +}; + +/// `pl.BlockSpec` for one pipeline operand. `null` slot ↔ `None` spec. +pub const PipeSpec = struct { + full_shape: []const i64, + dtype: DType, + /// Optional HBM source/result view to materialize at each DMA use site. + source_reshape_shape: ?[]const i64 = null, + /// `null` ⇒ trivial windowing (the whole array). + block_shape: ?[]const BlockDim = null, + /// `null` only allowed with trivial windowing. + index_map: ?IndexMapFn = null, + index_map_ctx: ?*anyopaque = null, + /// Only `.vmem` supported. + memory_space: MemorySpace = .vmem, + + buffer_count: ?usize = null, + /// Tile that `_create_bounded_slice` rounds the DMA size up to. + bounded_slice_tiling: i32 = 1, + /// Pallas's `disable_bounds_checks`: skip OOB clamp, emit only `tpu.assume_multiple`. + bounded_slice_skip_clamp: bool = false, + + /// Optional custom DMA behavior for this operand. + transfer_strategy: ?BufferedRefStrategy = null, + /// Override the role infered from whether this spec appears + /// in `in_specs` or `out_specs` + buffer_type: ?BufferType = null, +}; + +pub const PipelineOpts = struct { + body: BodyFn, + body_ctx: ?*anyopaque = null, + /// Dynamic (`i32`) grid bounds. Comptime: pass `b.constant(.i32, N)`. + grid: []const Value, + /// One per HBM input ref (slot `null` <-> `None` spec). + in_specs: []const ?PipeSpec, + out_specs: []const ?PipeSpec, + trace_scopes: bool = true, +}; + +/// Emits `pltpu.emit_pipeline(body, grid=..., in_specs=..., out_specs=...)(*refs, scratches=...)`. +/// `refs` = `in_refs ++ out_refs`. Emits into `b.currentBlock()`. +pub fn emitPipeline( + b: *Builder, + refs: []const ?Value, + scratches: []const Value, + opts: PipelineOpts, +) FinishError!void { + std.debug.assert(opts.grid.len >= 1); + if (opts.grid.len > 4) @panic("emitPipeline: grids with rank > 4 unsupported"); + const n_in = opts.in_specs.len; + std.debug.assert(refs.len == n_in + opts.out_specs.len); + + const a = b.arena.allocator(); + + var brefs: std.ArrayList(BufferedRef) = try .initCapacity(a, opts.in_specs.len + opts.out_specs.len); + for (opts.in_specs, 0..) |maybe_spec, i| { + if (maybe_spec) |spec| { + std.debug.assert(refs[i] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .input, refs[i].?, i)); + } + } + for (opts.out_specs, 0..) |maybe_spec, j| { + if (maybe_spec) |spec| { + std.debug.assert(refs[n_in + j] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .output, refs[n_in + j].?, n_in + j)); + } + } + + // Whole pipeline lives in a `tpu.region`. + const region_block = mlir.Block.init(&.{}, &.{}); + b.pushBlock(region_block); + + for (brefs.items) |*br| br.allocate(b); + + const num_steps = gridSize(b, opts.grid); + const c0_i32 = b.constant(.i32, 0); + const c1_i32 = b.constant(.i32, 1); + { + var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); + { + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .input or br.buffer_type == .input_output)) br.syncCopyIn(b); + + const num_stages = blk: { + var m: usize = 2; + for (brefs.items) |br| if (br.buffer_count > m) { + m = br.buffer_count; + }; + break :blk m; + }; + var sched0 = Scheduler.init(b, c0_i32, opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..num_stages - 1) |step| { + if (opts.trace_scopes) { + var buf: [32]u8 = undefined; + b.traceStart(10, std.fmt.bufPrint(&buf, "ep_initialize_{d}", .{step}) catch unreachable); + } + for (brefs.items) |*br| sched0.initializeStep(b, br, step); + if (opts.trace_scopes) b.traceStop(); + } + + // iter_args: each bref's slots in spec order (see appendSlotInits), then grid indices. + var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); + for (brefs.items) |*br| br.appendSlotInits(b, &inits); + for (0..opts.grid.len) |_| inits.appendAssumeCapacity(c0_i32.inner); + const n_slots = inits.items.len - opts.grid.len; + + const i32_ty = b.scalarTy(.i32); + const block_types = a.alloc(*const mlir.Type, inits.items.len + 1) catch unreachable; + const block_locs = a.alloc(*const mlir.Location, inits.items.len + 1) catch unreachable; + for (block_types) |*t| t.* = i32_ty; + for (block_locs) |*l| l.* = b.loc(); + const for_body = mlir.Block.init(block_types, block_locs); + b.pushBlock(for_body); + + const iv: Value = .{ .inner = for_body.argument(0), .kernel = b }; + { + var cur: usize = 1; + for (brefs.items) |*br| cur = br.bindSlotsAt(b, for_body, cur); + std.debug.assert(cur == n_slots + 1); + } + var grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + grid_indices_buf[i] = .{ .inner = for_body.argument(@intCast(n_slots + 1 + i)), .kernel = b }; + } + + var sched = Scheduler.init(b, iv, opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..opts.grid.len) |i| sched.indices_storage[i] = grid_indices_buf[i]; + sched.recomputeDerivedIndices(b); + + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.copyIn(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.waitIn(b, br); + } + // Emit slot `rem` ops BEFORE the body; body re-emits slice/squeeze per use. + var current_refs = a.alloc(RefWindow, refs.len) catch unreachable; + for (current_refs) |*r| r.* = .{}; + for (brefs.items) |*br| { + if (br.is_trivial or !br.isBuffered()) { + current_refs[br.spec_index] = .{ .buf = br.window_ref }; + } else { + const active = switch (br.buffer_type) { + .input => br.wait_in_slot.?, + .output, .input_output => br.copy_out_slot.?, + }; + current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.constant(.i32, br.buffer_count)) }; + } + } + + if (opts.trace_scopes) b.traceStart(10, "ep_run_kernel"); + try opts.body(b, sched.indices_storage[0..opts.grid.len], current_refs, scratches, opts.body_ctx); + if (opts.trace_scopes) b.traceStop(); + + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.copyOut(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.waitOut(b, br); + } + + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.advanceSlots(b, br); + } + var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); + for (brefs.items) |*br| br.appendSlotYields(b, &yields); + const next_indices = nextIndexND(b, sched.indices(), opts.grid); + for (next_indices) |idx| yields.appendAssumeCapacity(idx.inner); + _ = scf.yield(b.ctx, yields.items, b.loc()).appendTo(for_body); + b.popBlock(); + + const for_op = scf.for_(b.ctx, c0_i32.inner, num_steps.inner, c1_i32.inner, inits.items, for_body, .{}, b.loc()); + _ = for_op.appendTo(b.currentBlock()); + var k: usize = 0; + for (brefs.items) |*br| k = br.readbackSlots(b, for_op, k); + var final_grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + final_grid_indices_buf[i] = .{ .inner = for_op.result(@intCast(n_slots + i)), .kernel = b }; + } + + // prev_index BEFORE Scheduler.init so common arithmetic CSEs with `step`. + const final_idx = prevIndexND(b, final_grid_indices_buf[0..opts.grid.len], opts.grid); + var sched_fin = Scheduler.init(b, b.subi(num_steps, c1_i32), opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..opts.grid.len) |i| sched_fin.indices_storage[i] = final_idx[i]; + sched_fin.recomputeDerivedIndices(b); + for (brefs.items) |*br| sched_fin.finalize(b, br); + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .output or br.buffer_type == .input_output)) br.syncCopyOut(b); + + when.close(); + } + } + + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); +} + +pub const BufferType = enum { input, output, input_output }; + +/// VMEM double-buffer + DMA semaphores for one pipeline operand. +pub const BufferedRef = struct { + spec: PipeSpec, + buffer_type: BufferType, + buffer_count: usize, + is_trivial: bool, + /// Index into `refs` / `current_refs`. + spec_index: usize, + src: Value, + /// Block shape with `.squeezed` dims removed. + block_compact: []const i64, + /// Indices into `spec.block_shape` parallel to `block_compact`. + block_keep: []const usize, + + /// `memref` (or full shape if trivial). + window_ref: Value = undefined, + sem_recvs: ?Value = null, + sem_sends: ?Value = null, + + // Slot iter_arg state — see appendSlotInits for init values. + copy_in_slot: ?Value = null, + wait_in_slot: ?Value = null, + copy_out_slot: ?Value = null, + wait_out_slot: ?Value = null, + slot: ?Value = null, + + fn create(b: *Builder, spec: PipeSpec, bt: BufferType, src: Value, spec_index: usize) BufferedRef { + const a = b.arena.allocator(); + const buffer_type = spec.buffer_type orelse bt; + if (spec.memory_space != .vmem) @panic("emitPipeline: only VMEM operands supported"); + const trivial = specHasTrivialWindowing(spec); + var bc: usize = if (spec.buffer_count) |c| c else 2; + if (spec.buffer_count == null and trivial) bc = 1; + if (buffer_type != .input and bc > 2) @panic("emitPipeline: output and input_output buffer_count > 2 unsupported"); + + const block_len = if (spec.block_shape) |bs| bs.len else 0; + var keep = std.ArrayList(usize).initCapacity(a, block_len) catch unreachable; + var compact = std.ArrayList(i64).initCapacity(a, block_len) catch unreachable; + if (spec.block_shape) |bs| { + for (bs, 0..) |bd, di| switch (bd) { + .blocked => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .bounded_slice => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .squeezed => {}, + }; + } + return .{ + .spec = spec, + .buffer_type = buffer_type, + .buffer_count = bc, + .is_trivial = trivial, + .spec_index = spec_index, + .src = src, + .block_compact = compact.items, + .block_keep = keep.items, + }; + } + + fn isBuffered(self: *const BufferedRef) bool { + return self.buffer_count > 0; + } + + fn allocate(self: *BufferedRef, b: *Builder) void { + const a = b.arena.allocator(); + const vmem = MemorySpace.vmem.attribute(b.ctx); + const elem = self.spec.dtype.toMlir(b.ctx); + if (self.is_trivial) { + const ty = mlir.Type.memRef(elem, self.spec.full_shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } else { + if (self.block_compact.len == 1) @panic("emitPipeline: 1-D flat block buffers unsupported (TODO)"); + var shape = a.alloc(i64, self.block_compact.len + 1) catch unreachable; + shape[0] = @intCast(self.buffer_count); + @memcpy(shape[1..], self.block_compact); + const ty = mlir.Type.memRef(elem, shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } + if (!self.is_trivial) { + const sem_arr_ty = dmaSemRefTy(b, self.buffer_count); + if (self.buffer_type == .input or self.buffer_type == .input_output) self.sem_recvs = semAllocOp(b, sem_arr_ty); + if (self.buffer_type == .output or self.buffer_type == .input_output) self.sem_sends = semAllocOp(b, sem_arr_ty); + } + } + + fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + const c0 = b.constant(.i32, 0).inner; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(c0); + } else { + out.appendAssumeCapacity((self.copy_in_slot orelse b.constant(.i32, 0)).inner); + out.appendAssumeCapacity(c0); + } + }, + .output => { + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, + .input_output => { + out.appendAssumeCapacity( + (self.copy_in_slot orelse b.constant(.i32, 0)).inner, + ); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, + } + } + + fn nSlots(self: *const BufferedRef) usize { + return switch (self.buffer_type) { + .input => if (self.is_trivial) 1 else 2, + .output => 2, + .input_output => 4, + }; + } + + fn bindSlotsAt(self: *BufferedRef, b: *Builder, body: *mlir.Block, start: usize) usize { + var k = start; + const arg = struct { + fn f(b_: *Builder, body_: *mlir.Block, i: *usize) Value { + const v: Value = .{ .inner = body_.argument(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = arg(b, body, &k); + } else { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + } + }, + .output => { + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, + .input_output => { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, + } + return k; + } + + fn appendSlotYields(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + _ = b; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(self.slot.?.inner); + } else { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + } + }, + .output => { + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, + .input_output => { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, + } + } + + fn readbackSlots(self: *BufferedRef, b: *Builder, for_op: *mlir.Operation, start: usize) usize { + var k = start; + const take = struct { + fn f(b_: *Builder, op: *mlir.Operation, i: *usize) Value { + const v: Value = .{ .inner = op.result(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = take(b, for_op, &k); + } else { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + } + }, + .output => { + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, + .input_output => { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, + } + return k; + } + + fn computeIndex(self: *const BufferedRef, b: *Builder, indices: []const Value) []const BlockIndex { + if (self.spec.index_map) |im| return im(b, indices, self.spec.index_map_ctx); + @panic("emitPipeline: index_map required for non-trivial windowing"); + } + + /// Per `block_shape` dim: `{base, size?}`. `size == null` ⇒ static-full. + /// `static_zero_start` enables prologue shortcut (Python-int 0 start folds). + const DmaSlice = struct { base: []Value, size: []?Value }; + fn getDmaSlice(self: *const BufferedRef, b: *Builder, indices: []const Value, static_zero_start: bool) DmaSlice { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const bidx = self.computeIndex(b, indices); + std.debug.assert(bidx.len == bs.len); + var base = a.alloc(Value, bs.len) catch unreachable; + var size = a.alloc(?Value, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + switch (bd) { + .squeezed => { + base[d] = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: squeezed dim must get a scalar index"), + }; + size[d] = null; + }, + .blocked => |n| { + const idx = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: blocked dim must get a scalar index"), + }; + base[d] = if (n == 1) idx else idx.mul(@as(i32, @intCast(n))); + const dim_size = self.spec.full_shape[d]; + if (@mod(dim_size, n) == 0) { + size[d] = null; + } else { + const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; + const clamped = @min(n, dim_size); + const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.constant(.i32, sz_const).inner, @as(i32, @intCast(tiling)), b.loc())); + } + }, + .bounded_slice => { + const sl = switch (bidx[d]) { + .scalar => @panic("emitPipeline: bounded_slice dim must get a `.slice` index from index_map"), + .slice => |s| s, + }; + const tiling: i32 = self.spec.bounded_slice_tiling; + const dim_size = self.spec.full_shape[d]; + base[d] = sl.start; + if (self.spec.bounded_slice_skip_clamp) { + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); + continue; + } + // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. + if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { + const is_oob = b.cmpi(.sgt, sl.size, b.constant(.i32, dim_size)); + const sz = b.select(is_oob, b.constant(.i32, dim_size), sl.size); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + continue; + } + const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.constant(.i32, dim_size)); + const rounded = blk: { + if (tiling == 1) break :blk b.subi(b.constant(.i32, dim_size + 1), sl.start); + // lax sign-fixup for `jnp.remainder`. + const m = b.lift(tiling); + const c0 = b.constant(.i32, 0); + const rem_arg = b.subi(b.constant(.i32, dim_size), sl.start); + const r = b.remsi(rem_arg, m); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + const fixup = b.andi(r_neg, r_nz); + const r_plus = b.addi(r, m); + const floormod = b.select(fixup, r_plus, r); + const round_down = b.subi(rem_arg, floormod); + break :blk b.addi(round_down, m); + }; + const sz = b.select(is_oob, rounded, sl.size); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + }, + } + } + return .{ .base = base, .size = size }; + } + + fn sliceSrc(self: *const BufferedRef, b: *Builder, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const src = if (self.spec.source_reshape_shape) |shape| b.memRefReshape(self.src, shape) else self.src; + var base = a.alloc(Value, bs.len) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, bs.len) catch unreachable; + var result_shape = a.alloc(i64, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + base[d] = dma.base[d]; + if (dma.size[d]) |sz| { + result_shape[d] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[d] = switch (bd) { + .squeezed => 1, + .blocked => |n| n, + .bounded_slice => unreachable, + }; + } + } + const sliced = b.memRefSlice(src, base, result_shape, dyn.items); + if (self.block_keep.len == bs.len) return sliced; + + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + for (self.block_keep, 0..) |sd, ci| squeezed_shape[ci] = result_shape[sd]; + return b.memRefSqueeze(sliced, squeezed_shape); + } + + /// Slices `alloca[slot, 0…]` (non-squeezed dims) then squeezes the slot dim. + fn sliceWindowAt(self: *const BufferedRef, b: *Builder, slot: Value, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const c0 = b.constant(.i32, 0); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, self.block_keep.len) catch unreachable; + base[0] = slot; + result_shape[0] = 1; + for (self.block_keep, 0..) |sd, ci| { + base[ci + 1] = c0; + if (dma.size[sd]) |sz| { + result_shape[ci + 1] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[ci + 1] = switch (bs[sd]) { + .blocked => |n| n, + else => unreachable, + }; + } + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, dyn.items); + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + @memcpy(squeezed_shape, result_shape[1..]); + return b.memRefSqueeze(sliced, squeezed_shape); + } + + fn semAt(b: *Builder, sem_array: Value, i: Value) Value { + const sliced = b.memRefSlice(sem_array, &.{i}, &.{1}, &.{}); + return b.memRefSqueeze(sliced, &.{}); + } + + fn currentRef(self: *BufferedRef, b: *Builder) Value { + if (self.is_trivial or !self.isBuffered()) return self.window_ref; + const slot = switch (self.buffer_type) { + .output => b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)), + .input => b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)), + }; + const a = b.arena.allocator(); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + const c0 = b.constant(.i32, 0); + base[0] = slot; + result_shape[0] = 1; + for (self.block_compact, 0..) |n, ci| { + base[ci + 1] = c0; + result_shape[ci + 1] = n; + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, self.block_compact); + } + + fn syncCopyIn(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.src, self.window_ref, sem, .{}); + b.waitDma2(sem, self.src, self.window_ref, .{}); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + fn syncCopyOut(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.window_ref, self.src, sem, .{}); + b.waitDma2(sem, self.window_ref, self.src, .{}); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem + // before get_dma_slice; wait_in does it after. + + fn defaultCopyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + const slot = b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, static_zero_start); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn defaultWaitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + const dma = self.getDmaSlice(b, indices, false); + const slot = b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } + + fn defaultCopyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); + const slot = b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, false); + // Pallas evaluates `src` (VMEM window) before `dst` (HBM). + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn defaultWaitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count <= 1) return; + const slot = b.remui(self.wait_out_slot.?, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, false); + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } + + fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + if (self.is_trivial) return; + + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_in) |customCopyIn| { + customCopyIn(self, b, indices, slot_cumulative, static_zero_start, strategy.ctx); + return; + } + } + + self.defaultCopyIn(b, indices, slot_cumulative, static_zero_start); + } + + fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_in) |customWaitIn| { + customWaitIn(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitIn(b, indices); + } + + fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_out) |customCopyOut| { + customCopyOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultCopyOut(b, indices); + } + + fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_out) |customWaitOut| { + customWaitOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitOut(b, indices); + } + + pub fn sourceRef(self: *const BufferedRef) Value { + return self.src; + } + + pub fn windowRef(self: *const BufferedRef) Value { + return self.window_ref; + } + + pub fn copyInSlot(self: *const BufferedRef, b: *Builder, slot_cumulative: Value) Value { + return b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); + } + + pub fn waitInSlot(self: *const BufferedRef, b: *Builder) Value { + return b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); + } + + pub fn receiveSemaphoreAt(self: *const BufferedRef, b: *Builder, slot: Value) Value { + return semAt(b, self.sem_recvs.?, slot); + } + + pub fn copyOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.copy_out_slot.?, b.lift(buffer_count)); + } + + pub fn waitOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.wait_out_slot.?, b.lift(buffer_count)); + } + + pub fn sendSemaphoreAt( + self: *const BufferedRef, + b: *Builder, + slot: Value, + ) Value { + return semAt(b, self.sem_sends.?, slot); + } +}; + +/// Prologue/loop/epilogue copy+wait schedule. +const Scheduler = struct { + step: Value, + grid: []const Value, + num_steps: Value, + num_stages: usize, + trace_scopes: bool, + first_step: Value, + last_step: Value, + indices_storage: [4]Value = undefined, + prev_storage: [4]Value = undefined, + next_storage: [4]Value = undefined, + fetch_storage: [8][4]Value = undefined, + fetch_len: usize, + indices_len: usize, + + fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { + if (num_stages + 1 > 8) @panic("emitPipeline: buffer_count > 7 unsupported"); + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + var s: Scheduler = .{ + .step = step, + .grid = grid, + .num_steps = num_steps, + .num_stages = num_stages, + .trace_scopes = trace_scopes, + .first_step = b.cmpi(.eq, step, c0), + .last_step = b.cmpi(.eq, step, b.subi(num_steps, c1)), + .fetch_len = num_stages + 1, + .indices_len = grid.len, + }; + for (0..grid.len) |i| s.indices_storage[i] = c0; + s.recomputeDerivedIndices(b); + return s; + } + + fn indices(self: *Scheduler) []const Value { + return self.indices_storage[0..self.indices_len]; + } + fn prevIndices(self: *Scheduler) []const Value { + return self.prev_storage[0..self.indices_len]; + } + fn nextIndices(self: *Scheduler) []const Value { + return self.next_storage[0..self.indices_len]; + } + fn fetchIndices(self: *Scheduler, lookahead: usize) []const Value { + std.debug.assert(lookahead < self.fetch_len); + return self.fetch_storage[lookahead][0..self.indices_len]; + } + + fn recomputeDerivedIndices(self: *Scheduler, b: *Builder) void { + const prev = prevIndexND(b, self.indices(), self.grid); + const next = nextIndexND(b, self.indices(), self.grid); + for (0..self.indices_len) |i| { + self.prev_storage[i] = prev[i]; + self.next_storage[i] = next[i]; + self.fetch_storage[0][i] = self.indices_storage[i]; + self.fetch_storage[1][i] = next[i]; + } + var fetch = next; + var lookahead: usize = 2; + while (lookahead < self.fetch_len) : (lookahead += 1) { + fetch = nextIndexND(b, fetch, self.grid); + for (0..self.indices_len) |i| self.fetch_storage[lookahead][i] = fetch[i]; + } + } + + fn namedScope(self: *Scheduler, b: *Builder, name: []const u8) void { + if (self.trace_scopes) b.traceStart(10, name); + } + fn endScope(self: *Scheduler, b: *Builder) void { + if (self.trace_scopes) b.traceStop(); + } + + /// `~out_of_fetch` as `step < num_steps + (1 - buffer_count)`. + fn notOutOfFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered()); + const off: i32 = 1 - @as(i32, @intCast(br.buffer_count)); + return b.cmpi(.slt, self.step, b.addi(self.num_steps, b.lift(off))); + } + + fn notFirstStep(self: *Scheduler, b: *Builder) Value { + return b.cmpi(.ne, self.step, b.constant(.i32, 0)); + } + + fn indicesDiffer(b: *Builder, br: *const BufferedRef, ai: []const Value, ci: []const Value) Value { + const xa = br.computeIndex(b, ai); + const xc = br.computeIndex(b, ci); + std.debug.assert(xa.len == xc.len and xa.len > 0); + var acc: ?Value = null; + for (xa, xc) |x, y| { + const d: Value = switch (x) { + .scalar => |xv| b.cmpi(.ne, xv, y.scalar), + .slice => |xs| b.ori(b.cmpi(.ne, xs.start, y.slice.start), b.cmpi(.ne, xs.size, y.slice.size)), + }; + acc = if (acc) |p| b.ori(p, d) else d; + } + return acc.?; + } + + fn hasChanged(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.prevIndices()); + } + fn willChangeCurrent(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.nextIndices()); + } + fn willChangeFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + if (br.buffer_count < 2) return self.hasChanged(b, br); + return indicesDiffer(b, br, self.fetchIndices(br.buffer_count - 2), self.fetchIndices(br.buffer_count - 1)); + } + + fn advance(b: *Builder, slot: Value, pred: Value) Value { + return b.select(pred, b.addi(slot, b.constant(.i32, 1)), slot); + } + + fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { + if ((br.buffer_type != .input and br.buffer_type != .input_output) or !br.isBuffered() or br.is_trivial) return; + if (step + 1 >= br.buffer_count) return; + if (br.copy_in_slot == null) br.copy_in_slot = b.constant(.i32, 0); + + const pred = if (step == 0) self.first_step else blk: { + const block_changed = indicesDiffer(b, br, self.fetchIndices(step), self.fetchIndices(step - 1)); + break :blk b.andi(self.first_step, block_changed); + }; + const fetch_indices = self.fetchIndices(step); + if (step == 0) { + // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. + br.copyIn(b, fetch_indices, br.copy_in_slot.?, true); + } else { + var when = b.openIf(pred); + { + br.copyIn(b, fetch_indices, br.copy_in_slot.?, false); + when.close(); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); + } + + fn copyIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + var pred = b.andi(self.willChangeFetch(b, br), self.notOutOfFetch(b, br)); + if (br.isBuffered() and br.buffer_count < 2) pred = b.ori(pred, self.first_step); + if (br.buffer_type != .input and br.buffer_type != .input_output) return; + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_in"); + br.copyIn(b, self.fetchIndices(br.buffer_count - 1), br.copy_in_slot.?, false); + self.endScope(b); + when.close(); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); + } + + fn waitIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.hasChanged(b, br), self.first_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_in"); + if (br.buffer_type == .input or br.buffer_type == .input_output) br.waitIn(b, self.indices()); + self.endScope(b); + when.close(); + } + } + } + + fn copyOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_out"); + if (br.buffer_type == .output or br.buffer_type == .input_output) br.copyOut(b, self.indices()); + self.endScope(b); + when.close(); + } + } + br.copy_out_slot = advance(b, br.copy_out_slot.?, pred); + } + + fn waitOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.andi(self.hasChanged(b, br), self.notFirstStep(b)); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_out"); + // copy_out was issued the previous iteration. + if (br.buffer_type == .output or br.buffer_type == .input_output) br.waitOut(b, self.prevIndices()); + self.endScope(b); + when.close(); + } + } + br.wait_out_slot = advance(b, br.wait_out_slot.?, pred); + } + + fn finalize(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial or (br.buffer_type != .output and br.buffer_type != .input_output)) return; + var when = b.openIf(self.last_step); + { + self.namedScope(b, "ep_finalize"); + br.waitOut(b, self.indices()); + self.endScope(b); + when.close(); + } + } + + fn advanceSlots(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.buffer_type != .input and br.buffer_type != .input_output) return; + if (br.is_trivial) { + switch (br.buffer_type) { + .input => br.slot = advance(b, br.slot.?, self.last_step), + .input_output => br.wait_in_slot = advance(b, br.wait_in_slot.?, self.last_step), + .output => unreachable, + } + } else { + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + br.wait_in_slot = advance(b, br.wait_in_slot.?, pred); + } + } +}; + +pub fn memRefAlloca(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(memref.alloca(b.ctx, ty, &.{}, &.{}, .{}, b.loc())); +} + +pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(mosaic_tpu.sem_alloc(b.ctx, ty, b.loc())); +} + +/// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. +pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { + const shape: []const i64 = if (n) |m| &.{@intCast(m)} else &.{}; + return mlir.Type.memRef(mosaic_tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); +} + +fn gridSize(b: *Builder, grid: []const Value) Value { + var acc = grid[0]; + for (grid[1..]) |g| acc = b.muli(acc, g); + return acc; +} + +fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Value { + var out: [4]Value = undefined; + const c0 = b.constant(.i32, 0); + for (indices, grid, 0..) |i, g, pos| { + out[pos] = if (g.asConstantInt() == @as(?i64, 1)) c0 else i; + } + return out; +} + +fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("nextIndexND OOM"); + const c0 = b.constant(.i32, 0); + var carry: Value = b.cmpi(.eq, c0, c0); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const inc = b.select(carry, b.addi(indices[pos], b.constant(.i32, 1)), indices[pos]); + carry = b.cmpi(.eq, inc, grid[pos]); + out[pos] = b.select(carry, c0, inc); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; +} + +fn prevIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("prevIndexND OOM"); + const c0 = b.constant(.i32, 0); + var borrow: Value = b.cmpi(.eq, c0, c0); + const cm1 = b.constant(.i32, -1); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const dec = b.select(borrow, b.subi(indices[pos], b.constant(.i32, 1)), indices[pos]); + borrow = b.cmpi(.eq, dec, cm1); + out[pos] = b.select(borrow, b.subi(grid[pos], b.constant(.i32, 1)), dec); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; +} + +/// `null` block_shape, or every dim is `.blocked(full_dim)`. +fn specHasTrivialWindowing(spec: PipeSpec) bool { + const bs = spec.block_shape orelse return true; + if (bs.len != spec.full_shape.len) return false; + for (bs, spec.full_shape) |bd, fs| switch (bd) { + .blocked => |n| if (n != fs) return false, + else => return false, + }; + return true; +} + +test { + std.testing.refAllDecls(@This()); +} + +test "trivial input-output ref is copied in and out" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + builder.registerDialects(registry); + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.init(std.testing.allocator, ctx, "trivial_input_output", &.{ + .{ + .name = "state", + .kind = .{ .ref = .{ + .shape = &.{ 8, 128 }, + .dtype = .bf16, + .memory_space = .hbm, + } }, + }, + }, &.{}); + defer b.deinit(); + + const noOpBody = struct { + fn call( + _: *Builder, + _: []const Value, + _: []const RefWindow, + _: []const Value, + _: ?*anyopaque, + ) FinishError!void {} + }.call; + + try emitPipeline(&b, &.{b.arg(0)}, &.{}, .{ + .body = noOpBody, + .grid = &.{b.constant(.i32, 1)}, + .in_specs = &.{.{ + .full_shape = &.{ 8, 128 }, + .dtype = .bf16, + .buffer_type = .input_output, + }}, + .out_specs = &.{}, + .trace_scopes = false, + }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + // One enqueue/wait pair for copy_in, one for copy_out. + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.enqueue_dma")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.wait_dma")); +} diff --git a/dsl/tcl/BUILD.bazel b/dsl/tcl/BUILD.bazel new file mode 100644 index 00000000..096a3a67 --- /dev/null +++ b/dsl/tcl/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/tcl/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/tcl", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md new file mode 100644 index 00000000..b01b7189 --- /dev/null +++ b/dsl/tcl/README.md @@ -0,0 +1,170 @@ +# TCL kernels in Zig + +`dsl/tcl` is a Zig front end for Furiosa's TCL (Tensor Contraction +Language), shaped after the Python `furiosa.tcl` DSL: a kernel declares named +axes and typed tensors, opens tensor operations that `fetch`, `contract`, run +VE instructions and `commit`, and returns tensors. The `Builder` emits the +native `tcl` MLIR dialect (`mlir/dialects/tcl`), and +`zml.kernel.tcl.Kernel` turns it into a `furiosa.tcl` custom call that the +Furiosa plugin exports to TCL source and compiles with `furiosa-tcc` as one +native unit. + +## A kernel + +`furiosa.tcl.examples.softmax`, in Zig: + +```zig +const zml = @import("zml"); +const tcl = zml.kernel.tcl; + +const Cfg = struct { b: i64, s: i64, d: i64 }; + +pub const Softmax = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"input"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const exped = x.subf(x.reduce(&.{D}, .maxf)).exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); +} + +// In a model: +const y = Softmax.call(.{ .input = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .b = 2, .s = 16, .d = 1024 }, +}).out; +``` + +The plugin compiles it to the same TCL the Python DSL prints: + +``` +def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16) -> [B, S, D]/bf16: + v1: [B, S, D]/bf16 = tk.ReduceByVe( + %0 = read(v0, typecast_to=f32) + %1 = ve.exec(max_f t[D] %0) + %2 = ve.exec(%0 - %1) + %3 = ve.exec(exp %2) + %4 = ve.exec(+ t[D] %3) + %5 = ve.exec(%3 / %4) + write.to([B, S, D]/bf16) + ) + return v1 +``` + +## Python to Zig + +| `furiosa.tcl` | `dsl/tcl` | +|------------------------------------------------|-------------------------------------------------------| +| `x: tcl.Tensor[tcl.bf16, [B, S, D]]` | `.x = .{ .dtype = .bf16, .axes = &.{ B, S, D } }` | +| `x: tcl.Dram[tcl.bf16, [tcl.Broadcast], [M, N]]` | `.x = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }` | +| composite axis `C: Axis = (A * B)` | `b.axis("C", b.expr(.mul, A, B))` | +| `@tcl.kernel(config={...}, auto_config={...})` | `b.compilerConfig(.{ ... })`, `b.autoConfig(.{ ... })` | +| `@tcl.tensor_operation` | `const op = b.tensorOperation(.{})` | +| `fetch.typecast(x, tcl.f32)` | `op.fetch(x, .{ .typecast_to = .f32 })` | +| `fetch.broadcast(x, ...)`, `fetch.table_lookup` | `.broadcast_to = &.{...}`, `.table_lookup = table` | +| `fetch.pad(x, axis=L, left=1, right=1)` | `.pad = &.{.{ .axis = L, .left = 1, .right = 1 }}` | +| `fetch.slide(x, axis=L, frame_axis=Lf, window_axis=Lw)` | `.slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}` | +| `operation.contract(lhs, rhs, to=[B, M, N])` | `op.contract(lhs, rhs, &.{ B, M, N })` | +| `operation.subf(a, b)`, `operation.exp(a)` | `a.subf(b)`, `a.exp()`, or `op.ve(.subf, .{ a, b })` | +| `operation.reduce(x, axes=[D], op=tcl.Maxf)` | `x.reduce(&.{D}, .maxf)` | +| `operation.where(x > 0, a, b)` | `op.where(x, .gt, 0, a, b)` | +| `operation.to_fp(x, 15)`, `to_fxp(x, 15)` | `x.toFp(15)`, `x.toFxp(15)` | +| `compiler_hints=tcl.context(layout={Chip: ...})` | `.context = .{ .layout = .{ .chip = .broadcast } }` | +| `commit.typecast(v, tcl.bf16)` / `commit.permute` | `op.commit(v, .{ .dtype = .bf16, .axes = ... })` | +| `tcl.reshape`, `transmute`, `concat`, `slice` | `b.reshape`, `b.transmute`, `b.concat`, `b.slice` | +| `tcl.gather`, `scatter`, `arange`, `vector`, `all_gather` | `b.gather`, `b.scatter`, `b.arange`, `b.vector`, `b.allGather` | +| `for i in tcl.range(0, n):` (carried `acc`) | `var loop = b.openFor(n, .{acc})` ... `try loop.yield(.{new_acc})`, then `loop.results` | +| `x[i]`, `out[i] = v`, `tcl.scratchpad()` | `b.indexRead(x, loop.iv)`, `b.indexWrite(out, loop.iv, v)`, `b.scratchpad` | +| `core.reduce_max_i32(t)`, `core.eval_expr(...)` | `b.reduceMaxI32(t)`, `b.symExpr(.div, n, 64)` | +| `return out` | `b.ret(&.{out})` | + +* **Axes are static.** `b.axis(name, size)` declares `name: Axis = size`, + with an integer or an `Expr` over other axes; axis names start with an + uppercase letter. `furiosa-tcc` needs every axis bound. Kernel inputs and + outputs must match the XLA shapes axis by axis (row-major), which `call` + checks. +* **Loops.** Put the body in a Zig `{ ... }` block right after `openFor`, + ending with `try loop.yield(...)`. `loop` stays outside, so `loop.results` + stays readable after the block. +* **DRAM tensors.** A `.dram` argument comes back as its `as_logical` view; + `b.asDram(t, .{...})` returns a result in a DRAM mapping. +* **Multi-chip kernels.** A kernel that places data or work on chips (a + chip-split DRAM tensor, a `.context`, `allGather`) is compiled by the plugin + for all the chips of its SPMD program (2 or 4), and runs as one launch + across the partitions; other kernels stay per chip. Its arguments and + results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis + `C`, sized to the chip count, across chips (each partition holds its share), + the default `.broadcast` keeps a copy per chip. +* **Convolutions** pad and slide the input in its read: a slide replaces its + axis by `window_axis, frame_axis`, then `contract` with the filter. On its + own (without a slide) the padding is dropped. +* **The mainstream rule.** As in the Python DSL, instructions are pipelined: + `commit` must take the last instruction, there is at most one contraction + and it comes before VE instructions. Reads are hoisted in front. +* **VE registers are f32 or i32**, decided by the instruction. Scalar + operands become `f32`/`i32` constants (`x.mulf(0.5)`). +* **Tactics are guessed** like `@tcl.tensor_operation` (`EinsumByDpe`, + `ReduceByVe`, `Interleaving`, ...); `b.tensorOperation(.{ .tactic = ... })` + overrides it. +* **Tensors passed directly** to an instruction are read without modifiers. +* **Constant outputs.** `furiosa-tcc` 2026.3 fails ("No producer") on a + function returning two input-independent tensors, such as `arange` and + `vector`. +* **Loops** run on the device. `x[i]` and `acc[i] = v` address the outermost + axis, and the body must index something with `i`: that bounds the trip + count. A dynamic trip count must be computed in the kernel (`reduceMaxI32`, + `symExpr`); one read straight from an argument adds a CPU node the plugin + rejects. + +## Good practices on RNGD + +The TCL CPU interpreter (`furiosa.torch.TclModule`) accepts several programs +that compute garbage on the hardware: check kernels on RNGD. + +* **One VE stream per tensor operation.** Each instruction takes the previous + result and one side operand (a read or a constant). A select whose branches + are both constants must test the operation's first read. +* **Outer broadcasts** (`[B, H, V] * [B, H, K]` to `[B, H, V, K]`, tactic + `EinsumByVe`) need a shared leading axis, and the read with more axes first. +* **Contraction epilogues** read their extra tensors before the contraction + and combine the contraction result first: `contract(q, k).addf(bias)`. +* **Conversions.** `typecast_to` converts floats only; `toFp(n)` of an i32 + yields `x / 2^31` whatever `n`. +* **Update caches in place.** `scatter(rows, idx, axis, .{ .init = cache })` + on a kernel argument shares its buffer with the result (a whole-cache copy + otherwise), so return it as an aliased output. +* **Few axes, few operations.** tcc's tactic search grows with the number and + extents of axes, and more than linearly with the number of operations in a + function: merge axes before contractions (`[N, H, G*Q, C]`, not + `[N, H, G, Q, P, C]`) and set `.lowering_mode = "Heuristic"` above a few + tokens. +* **Prefer one-shot graphs to hand-written online softmax.** Furiosa's + production attention gathers the context, contracts, masks with a select, + runs the softmax as one `ReduceByVe` and contracts again; gathers take + `valid_length` to skip padding, and padding indices are spread + (`idx | (pos % n)`) rather than all pointing at one row. +* **Furiosa's kernels** in the SDK (`furiosa/kernels/`) are the reference for + layouts, tile sizes and compiler configs: `common/attention.py` (attention + and decode/prefill configs), `common/topk.py` and + `common/blockwise_moe_index.py` (routing on the device), + `qwen3_moe/optimized/` (blockwise MoE for prefill, activation-stationary MoE + for decode), `exaone_moe/k_exaone_w4fa16kv16/flash_attention.py` (chunked + softmax over a dynamic trip count). + +## The custom call + +`ops.tcl` emits `stablehlo.custom_call @furiosa.tcl` with the printed module +as its backend config. The plugin parses it, checks the function signature +against the operand and result shapes, exports TCL source and compiles it +with the rest of the program's units. With `--xla_dump_to=`, +`*.furiosa-unit-sources.txt` holds each unit's TCL. diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig new file mode 100644 index 00000000..f86d338b --- /dev/null +++ b/dsl/tcl/builder.zig @@ -0,0 +1,1429 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const tcl = @import("mlir/dialects/tcl"); + +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const Tactic = tcl.Tactic; +pub const ReduceMode = tcl.ReduceMode; +pub const Predicate = tcl.Predicate; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, tcl }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); +} + +/// A named TCL axis with its extent, from `Builder.axis`. +pub const Axis = struct { + name: []const u8, + size: i64, +}; + +/// A TCL expression over axes and integers, for composite axes +/// (`C: Axis = (A * B)`) and DRAM mappings (`D @ Dp`, `F / 2`, `Broadcast`). +/// `Builder.expr` makes the nodes. +pub const Expr = union(enum) { + int: i64, + axis: Axis, + broadcast, + node: *const Node, + + pub const Kind = enum { add, sub, mul, div, rem, exact_div, stride, modulo, padding, resize, pair }; + pub const Node = struct { kind: Kind, lhs: Expr, rhs: Expr }; + + /// The value of integer arithmetic over axes; null for mappings. + pub fn value(self: Expr) ?i64 { + return switch (self) { + .int => |n| n, + .axis => |a| a.size, + .broadcast => null, + .node => |n| { + const l = n.lhs.value() orelse return null; + const r = n.rhs.value() orelse return null; + return switch (n.kind) { + .add => l + r, + .sub => l - r, + .mul => l * r, + .div, .exact_div => if (r == 0) null else @divFloor(l, r), + .rem => if (r == 0) null else @mod(l, r), + else => null, + }; + }, + }; + } +}; + +/// `dram(chip | inner)/T`: a tensor in an explicit DRAM mapping, Python's +/// `tcl.Dram[T, chip, inner, tcl.original[...]]`. `chip` is `.broadcast` +/// (a copy per chip) or the axes split across chips; `inner` defaults to the +/// tensor's other axes. +pub const Dram = struct { + chip: []const Expr = &.{.broadcast}, + inner: ?[]const Expr = null, + original: []const Axis = &.{}, + + fn shards(self: Dram) i64 { + var n: i64 = 1; + for (self.chip) |e| if (e == .axis) { + n *= e.axis.size; + }; + return n; + } + + fn isChipAxis(self: Dram, a: Axis) bool { + for (self.chip) |e| if (e == .axis and std.mem.eql(u8, e.axis.name, a.name)) return true; + return false; + } +}; + +/// `@ context(operator = {...}, heuristic_hint = {...})`, Python's +/// `tcl.context(layout=..., heuristic_hint=...)`. +pub const Context = struct { + layout: ?Layout = null, + heuristic_hint: []const Hint = &.{}, + + /// `{Chip: ..., Cluster: ..., Split: [...]}`: `chip` and `cluster` are an + /// `.axis` or `.broadcast`. + pub const Layout = struct { + chip: Expr, + cluster: ?Expr = null, + split: []const Axis = &.{}, + }; + + pub const Hint = struct { + name: []const u8, + value: union(enum) { int: i64, float: f64, boolean: bool, axis: Axis }, + }; +}; + +/// The VE (vector engine) instructions, named like `furiosa.tcl.operation`. +pub const VeOp = enum { + exp, + neg_exp, + sqrt, + tanh, + sigmoid, + erf, + log, + sin, + cos, + negf, + absf, + addi, + addi_sat, + subi, + subi_sat, + mul_q31, + muli, + shl, + shl_sat, + shr_logical, + shr_arith, + shr_arith_round, + bit_andi, + bit_ori, + bit_xori, + mini, + maxi, + absmini, + absmaxi, + addf, + subf, + mulf, + divf, + bit_andf, + bit_orf, + bit_xorf, + minf, + maxf, + absminf, + absmaxf, + reinterpret_as_float, + reinterpret_as_int, + to_float, + to_int, + + pub fn arity(self: VeOp) usize { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .reinterpret_as_float, .reinterpret_as_int, .to_float, .to_int => 1, + else => 2, + }; + } + + /// VE registers are f32 or i32, according to the instruction. + pub fn result(self: VeOp) DType { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .addf, .subf, .mulf, .divf, .bit_andf, .bit_orf, .bit_xorf, .minf, .maxf, .absminf, .absmaxf, .reinterpret_as_float, .to_float => .f32, + else => .i32, + }; + } +}; + +/// A graph-level tensor: a kernel argument, or the result of a tensor or +/// graph operation (`x: [A, B]/bf16` in TCL). +pub const Tensor = struct { + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + /// Chips a DRAM tensor is split across: each SPMD partition holds + /// 1/shards of it. + shards: i64 = 1, +}; + +/// A value inside a tensor operation (`%N` in TCL): the result of a read, +/// the contraction or a VE instruction. +pub const Value = struct { + op: *TensorOperation, + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + + pub fn unary(self: Value, comptime opcode: VeOp) Value { + return self.op.ve(opcode, .{self}); + } + + /// `rhs` is a `Value`, a `Tensor` (read on use) or a scalar literal. + pub fn binary(self: Value, comptime opcode: VeOp, rhs: anytype) Value { + return self.op.ve(opcode, .{ self, rhs }); + } + + pub fn reduce(self: Value, axes: []const Axis, mode: ReduceMode) Value { + return self.op.reduce(self, axes, mode); + } + + pub fn cumsum(self: Value, axes: []const Axis) Value { + return self.op.cumsum(self, axes); + } + + pub fn addf(self: Value, rhs: anytype) Value { + return self.binary(.addf, rhs); + } + pub fn subf(self: Value, rhs: anytype) Value { + return self.binary(.subf, rhs); + } + pub fn mulf(self: Value, rhs: anytype) Value { + return self.binary(.mulf, rhs); + } + pub fn divf(self: Value, rhs: anytype) Value { + return self.binary(.divf, rhs); + } + pub fn maxf(self: Value, rhs: anytype) Value { + return self.binary(.maxf, rhs); + } + pub fn minf(self: Value, rhs: anytype) Value { + return self.binary(.minf, rhs); + } + pub fn addi(self: Value, rhs: anytype) Value { + return self.binary(.addi, rhs); + } + pub fn subi(self: Value, rhs: anytype) Value { + return self.binary(.subi, rhs); + } + pub fn muli(self: Value, rhs: anytype) Value { + return self.binary(.muli, rhs); + } + pub fn exp(self: Value) Value { + return self.unary(.exp); + } + pub fn sqrt(self: Value) Value { + return self.unary(.sqrt); + } + pub fn tanh(self: Value) Value { + return self.unary(.tanh); + } + pub fn sigmoid(self: Value) Value { + return self.unary(.sigmoid); + } + pub fn log(self: Value) Value { + return self.unary(.log); + } + pub fn negf(self: Value) Value { + return self.unary(.negf); + } + pub fn toFloat(self: Value) Value { + return self.unary(.to_float); + } + pub fn toInt(self: Value) Value { + return self.unary(.to_int); + } + pub fn toFp(self: Value, int_width: u5) Value { + return self.op.toFp(self, int_width); + } + pub fn toFxp(self: Value, int_width: u5) Value { + return self.op.toFxp(self, int_width); + } +}; + +/// One TCL tensor operation, `tk.(...)`: reads, at most one +/// contraction, VE instructions, then `commit` writes its result tensor. +/// Like `@tcl.tensor_operation`, every instruction after the reads is +/// pipelined: `commit` takes the last one. +pub const TensorOperation = struct { + pub const Options = struct { + /// Guessed from the instructions like the Python DSL when null. + tactic: ?Tactic = null, + context: ?Context = null, + }; + + pub const FetchOptions = struct { + typecast_to: ?DType = null, + broadcast_to: ?[]const Axis = null, + subtraction: ?Tensor = null, + table_lookup: ?Tensor = null, + /// Padding along an axis, applied by a `slide` over it. + pad: []const Pad = &.{}, + /// Sliding windows: each replaces its axis by `window_axis, frame_axis`. + slide: []const Slide = &.{}, + }; + + pub const Slide = struct { + axis: Axis, + frame_axis: Axis, + window_axis: Axis, + /// The window before dilation; `window_axis.size` by default. + undilated_window: ?i64 = null, + stride: i64 = 1, + dilation: i64 = 1, + }; + + pub const Pad = struct { + axis: Axis, + left: i64 = 0, + right: i64 = 0, + fill: f64 = 0, + }; + + pub const CommitOptions = struct { + dtype: ?DType = null, + /// A permutation of the committed value's axes. + axes: ?[]const Axis = null, + }; + + b: *Builder, + tactic: ?Tactic, + context: ?Context, + body: *mlir.Block, + inputs: std.ArrayList(Tensor) = .empty, + plain_reads: std.ArrayList(struct { tensor: *const mlir.Value, value: Value }) = .empty, + first_compute: ?*mlir.Operation = null, + last: ?*const mlir.Value = null, + has_dpe: bool = false, + has_ve: bool = false, + has_reduce: bool = false, + has_select: bool = false, + + /// `read(t, typecast_to=..., broadcast_to=..., ...)`. + pub fn fetch(self: *TensorOperation, t: Tensor, opts: FetchOptions) Value { + const b = self.b; + const ctx = b.ctx; + var operands: [3]*const mlir.Value = .{ self.input(t), undefined, undefined }; + var n: usize = 1; + var fields: [6]mlir.NamedAttribute = undefined; + var nfields: usize = 0; + var dtype = t.dtype; + var axes = t.axes; + if (opts.subtraction) |s| { + operands[n] = self.input(s); + fields[nfields] = .named(ctx, "subtraction", .int(ctx, .i64, n)); + n += 1; + nfields += 1; + } + if (opts.table_lookup) |table| { + operands[n] = self.input(table); + fields[nfields] = .named(ctx, "table_lookup", .int(ctx, .i64, n)); + dtype = table.dtype; + n += 1; + nfields += 1; + } + if (opts.typecast_to) |to| { + fields[nfields] = .named(ctx, "typecast_to", .typeAttr(to.toMlir(ctx))); + dtype = to; + nfields += 1; + } + if (opts.pad.len > 0) { + const pads = b.arena.allocator().alloc(mlir.NamedAttribute, opts.pad.len) catch @panic("OOM"); + for (pads, opts.pad) |*dst, p| dst.* = .named(ctx, p.axis.name, .array(ctx, &.{ + .int(ctx, .i64, p.left), + .int(ctx, .i64, p.right), + .float(ctx, .f64, p.fill), + })); + fields[nfields] = .named(ctx, "pad", .dict(ctx, pads)); + nfields += 1; + } + if (opts.slide.len > 0) { + const slides = b.arena.allocator().alloc(mlir.NamedAttribute, opts.slide.len) catch @panic("OOM"); + var slid: std.ArrayList(Axis) = .empty; + slid.appendSlice(b.arena.allocator(), axes) catch @panic("OOM"); + for (slides, opts.slide) |*dst, sl| { + dst.* = .named(ctx, sl.axis.name, .dict(ctx, &.{ + .named(ctx, "undilated_window", .int(ctx, .i64, sl.undilated_window orelse sl.window_axis.size)), + .named(ctx, "frame_axis", b.axisAttr(sl.frame_axis)), + .named(ctx, "window_axis", b.axisAttr(sl.window_axis)), + .named(ctx, "stride", .int(ctx, .i64, sl.stride)), + .named(ctx, "dilation", .int(ctx, .i64, sl.dilation)), + })); + const at = for (slid.items, 0..) |a, i| { + if (std.mem.eql(u8, a.name, sl.axis.name)) break i; + } else std.debug.panic("tcl: {s}: slide axis {s} is not read", .{ b.name, sl.axis.name }); + slid.items[at] = sl.window_axis; + slid.insert(b.arena.allocator(), at + 1, sl.frame_axis) catch @panic("OOM"); + } + fields[nfields] = .named(ctx, "slide", .dict(ctx, slides)); + nfields += 1; + axes = slid.items; + } + if (opts.broadcast_to) |to| { + fields[nfields] = .named(ctx, "broadcast_to", b.axesAttr(to)); + axes = b.dupeAxes(to); + nfields += 1; + } + const options = if (nfields == 0) null else tcl.readOptionsAttr(ctx, fields[0..nfields]) catch + std.debug.panic("tcl: invalid read options for {s}", .{b.name}); + const op = tcl.read(ctx, operands[0..n], b.logicalType(dtype, axes), .{ .options = options }, b.loc()); + return self.emit(op, dtype, axes, true); + } + + /// `dpe.exec(lhs, rhs -> [to])`: shared axes missing from `to` are + /// contracted. Accumulates in f32 (float inputs) or i32. + pub fn contract(self: *TensorOperation, lhs: anytype, rhs: anytype, to: []const Axis) Value { + if (self.has_dpe or self.has_ve) std.debug.panic("tcl: one contraction per tensor operation, before VE instructions", .{}); + const l = self.lift(lhs, .f32); + const r = self.lift(rhs, .f32); + const dtype: DType = if (l.dtype.isFloat()) .f32 else .i32; + self.has_dpe = true; + const op = tcl.dpe(self.b.ctx, l.inner, r.inner, self.b.logicalType(dtype, to), self.b.loc()); + return self.emit(op, dtype, self.b.dupeAxes(to), false); + } + + /// `ve.exec(...)` of a unary or binary instruction. Operands are + /// `Value`s, `Tensor`s (read on use) or scalar literals. + pub fn ve(self: *TensorOperation, comptime opcode: VeOp, operands: anytype) Value { + if (comptime operands.len != opcode.arity()) @compileError("tcl: wrong operand count for " ++ @tagName(opcode)); + return self.veNamed(@tagName(opcode), opcode.result(), operands); + } + + /// `ve.exec(x to_f)`: Q-format fixed point to f32, Python's + /// `operation.to_fp`. + pub fn toFp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_f{d}", .{int_width}) catch unreachable, .f32, .{x}); + } + + /// `ve.exec(x to_i)`: f32 to Q-format fixed point, Python's + /// `operation.to_fxp`. + pub fn toFxp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_i{d}", .{int_width}) catch unreachable, .i32, .{x}); + } + + fn veNamed(self: *TensorOperation, opcode: []const u8, scalar: DType, operands: anytype) Value { + var values: [2]*const mlir.Value = undefined; + var axes: []const Axis = &.{}; + inline for (operands, 0..) |o, i| { + const v = self.lift(o, scalar); + values[i] = v.inner; + axes = self.b.unionAxes(axes, v.axes); + } + self.has_ve = true; + const ctx = self.b.ctx; + const code = tcl.veOpcodeAttr(ctx, opcode) catch std.debug.panic("tcl: unknown VE instruction {s}", .{opcode}); + const op = tcl.ve(ctx, values[0..operands.len], self.b.logicalType(scalar, axes), code, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `ve.exec( t[axes] x)`: removes `axes`. + pub fn reduce(self: *TensorOperation, x: anytype, axes: []const Axis, mode: ReduceMode) Value { + if (mode == .cumsum) return self.cumsum(x, axes); + const v = self.lift(x, .f32); + var kept: std.ArrayList(Axis) = .empty; + for (v.axes) |a| { + if (!containsAxis(axes, a)) kept.append(self.b.arena.allocator(), a) catch @panic("OOM"); + } + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, kept.items), mode, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, kept.items, false); + } + + /// `cumsum t[axes] x`, like `operation.cumsum`: the running sum of an i32 + /// value along `axes`, flattened in that order. + pub fn cumsum(self: *TensorOperation, x: anytype, axes: []const Axis) Value { + const v = self.lift(x, .i32); + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, v.axes), .cumsum, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, v.axes, false); + } + + /// `ve.exec(if cond threshold { yes } else { no })`, like + /// `operation.where(cond threshold, yes, no)`. + pub fn where(self: *TensorOperation, cond: anytype, pred: Predicate, threshold: anytype, yes: anytype, no: anytype) Value { + const ctx = self.b.ctx; + const c = self.lift(cond, .f32); + const float = isFloatOperand(yes) or isFloatOperand(no); + const scalar: DType = if (float) .f32 else .i32; + const t = self.lift(yes, scalar); + const f = self.lift(no, scalar); + const threshold_attr: *const mlir.Attribute = if (c.dtype.isFloat()) + .float(ctx, .f32, threshold) + else + .int(ctx, .i32, threshold); + const axes = self.b.unionAxes(self.b.unionAxes(c.axes, t.axes), f.axes); + self.has_ve = true; + self.has_select = true; + const op = tcl.ve_select(ctx, c.inner, t.inner, f.inner, self.b.logicalType(scalar, axes), pred, threshold_attr, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `write.to(...)`: closes the tensor operation and returns its result. + pub fn commit(self: *TensorOperation, v: Value, opts: CommitOptions) FinishError!Tensor { + const b = self.b; + if (self.last != v.inner) std.debug.panic("tcl: {s}: commit must take the last instruction", .{b.name}); + const axes = if (opts.axes) |a| b.dupeAxes(a) else v.axes; + if (!sameAxisSet(axes, v.axes)) std.debug.panic("tcl: {s}: commit can only permute axes", .{b.name}); + const dtype = opts.dtype orelse v.dtype; + _ = tcl.write(b.ctx, v.inner, b.loc()).appendTo(self.body); + + const inputs = try b.arena.allocator().alloc(*const mlir.Value, self.inputs.items.len); + for (inputs, self.inputs.items) |*dst, t| dst.* = t.inner; + const context = if (self.context) |c| b.contextAttr(c) else null; + const kernel = tcl.kernel(b.ctx, inputs, b.logicalType(dtype, axes), self.tactic orelse self.guessTactic(axes), self.body, .{ .context = context }, b.loc()); + try b.append(kernel); + return .{ .inner = kernel.result(0), .dtype = dtype, .axes = axes }; + } + + /// Python's `BuilderHistory.guess_tactic_kind`. + fn guessTactic(self: *const TensorOperation, output: []const Axis) Tactic { + if (self.has_dpe) return .einsum_by_dpe; + if (!self.has_ve) return .tensor_operation; + if (self.has_reduce) return .reduce_by_ve; + const in = self.inputs.items; + if (in.len > 0 and in[0].axes.len < output.len) return .einsum_by_ve; + if (!self.has_select and in.len == 2 and sameAxisSet(in[0].axes, in[1].axes)) return .interleaving; + return .elementwise; + } + + const Operand = struct { inner: *const mlir.Value, dtype: DType, axes: []const Axis }; + + fn lift(self: *TensorOperation, x: anytype, scalar: DType) Operand { + const T = @TypeOf(x); + if (T == Value) return .{ .inner = x.inner, .dtype = x.dtype, .axes = x.axes }; + if (T == Tensor) { + const r = self.plainRead(x); + return .{ .inner = r.inner, .dtype = r.dtype, .axes = r.axes }; + } + const ctx = self.b.ctx; + const op = switch (@typeInfo(T)) { + .comptime_int, .int => if (scalar == .f32) + arith.constant_float(ctx, @floatFromInt(x), .f32, self.b.loc()) + else + arith.constant_int(ctx, x, .int(ctx, .i32), self.b.loc()), + .comptime_float, .float => blk: { + if (scalar != .f32) std.debug.panic("tcl: float literal for an integer VE instruction", .{}); + break :blk arith.constant_float(ctx, x, .f32, self.b.loc()); + }, + else => @compileError("tcl: expected a Value, a Tensor or a scalar, got " ++ @typeName(T)), + }; + _ = op.appendTo(self.body); + return .{ .inner = op.result(0), .dtype = scalar, .axes = &.{} }; + } + + /// Registers `t` as a kernel operand. + fn input(self: *TensorOperation, t: Tensor) *const mlir.Value { + if (t.inner.type_().isA(tcl.LogicalType) == null) + std.debug.panic("tcl: {s}: tensor operations read logical tensors, not DRAM ones", .{self.b.name}); + for (self.inputs.items) |known| { + if (known.inner == t.inner) return t.inner; + } + self.inputs.append(self.b.arena.allocator(), t) catch @panic("OOM"); + return t.inner; + } + + fn plainRead(self: *TensorOperation, t: Tensor) Value { + for (self.plain_reads.items) |r| if (r.tensor == t.inner) return r.value; + const r = self.fetch(t, .{}); + self.plain_reads.append(self.b.arena.allocator(), .{ .tensor = t.inner, .value = r }) catch @panic("OOM"); + return r; + } + + /// Reads go before every other instruction, as TCL lists them. + fn emit(self: *TensorOperation, op: *mlir.Operation, dtype: DType, axes: []const Axis, is_read: bool) Value { + if (is_read and self.first_compute != null) { + self.body.insertOwnedOperationBefore(self.first_compute.?, op); + } else { + _ = op.appendTo(self.body); + if (!is_read and self.first_compute == null) self.first_compute = op; + self.last = op.result(0); + } + return .{ .op = self, .inner = op.result(0), .dtype = dtype, .axes = axes }; + } +}; + +fn isFloatOperand(x: anytype) bool { + return switch (@typeInfo(@TypeOf(x))) { + .comptime_float, .float => true, + .comptime_int, .int => false, + else => x.dtype.isFloat(), + }; +} + +fn containsAxis(axes: []const Axis, a: Axis) bool { + for (axes) |x| if (std.mem.eql(u8, x.name, a.name)) return true; + return false; +} + +fn sameAxisSet(a: []const Axis, b: []const Axis) bool { + if (a.len != b.len) return false; + for (a) |x| if (!containsAxis(b, x)) return false; + return true; +} + +/// Builds one TCL function, `def name(axes..., args...) -> results:`, as a +/// `func.func` of the native `tcl` dialect. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + axes: std.ArrayList(Declared) = .empty, + compiler_config: ?*const mlir.Attribute = null, + auto_config: ?*const mlir.Attribute = null, + fusible: bool = false, + func_op: ?*mlir.Operation = null, + entry: ?*mlir.Block = null, + args: []const Tensor = &.{}, + results: ?[]const Tensor = null, + /// Arguments `update` writes back into, by argument number. + updated: std.ArrayList(usize) = .empty, + /// Blocks being built inside the entry block (loop bodies), innermost last. + block_stack: common.BlockStack = .{}, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + _ = tcl.symbolAttr(ctx, name) catch std.debug.panic("tcl: `{s}` is not a TCL function name", .{name}); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = .init(.unknown(ctx)), + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + const Declared = struct { axis: Axis, def: ?Expr }; + + /// `A: Axis = size`, where `size` is an integer or an `Expr` over other + /// axes (`b.axis("C", b.expr(.mul, A, B))`). Axis names start with an + /// uppercase letter. + pub fn axis(self: *Builder, name: []const u8, size: anytype) Axis { + const def: ?Expr, const n: i64 = if (comptime @TypeOf(size) == Expr) + .{ size, size.value() orelse std.debug.panic("tcl: axis {s} needs an integer size", .{name}) } + else + .{ null, size }; + if (name.len == 0 or !std.ascii.isUpper(name[0]) or n <= 0) + std.debug.panic("tcl: invalid axis {s} = {d}", .{ name, n }); + _ = tcl.symbolAttr(self.ctx, name) catch std.debug.panic("tcl: invalid axis name {s}", .{name}); + for (self.axes.items) |d| if (std.mem.eql(u8, d.axis.name, name)) { + if (d.axis.size != n) std.debug.panic("tcl: axis {s} redeclared as {d} (was {d})", .{ name, n, d.axis.size }); + return d.axis; + }; + const a: Axis = .{ .name = self.arena.allocator().dupe(u8, name) catch @panic("OOM"), .size = n }; + self.axes.append(self.arena.allocator(), .{ .axis = a, .def = def }) catch @panic("OOM"); + return a; + } + + /// An `Expr` node: `lhs` and `rhs` are axes, integers or expressions. + pub fn expr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) Expr { + const node = self.arena.allocator().create(Expr.Node) catch @panic("OOM"); + node.* = .{ .kind = kind, .lhs = toExpr(lhs), .rhs = toExpr(rhs) }; + return .{ .node = node }; + } + + fn toExpr(x: anytype) Expr { + return switch (@TypeOf(x)) { + Expr => x, + Axis => .{ .axis = x }, + else => .{ .int = x }, + }; + } + + /// `@compiler_config({...})`, Python's `@tcl.kernel(config=...)`: an + /// anonymous struct written out as JSON, `.{ .lowering_mode = "Heuristic" }`. + pub fn compilerConfig(self: *Builder, config: anytype) void { + self.compiler_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid compiler config", .{self.name}); + } + + /// Lets the plugin compile the function into the unit of the work around + /// it instead of a unit of its own; the unit's compiler config applies. + pub fn fuse(self: *Builder) void { + self.fusible = true; + } + + /// `@auto_config({...})`, Python's `@tcl.kernel(auto_config=...)`. + pub fn autoConfig(self: *Builder, config: anytype) void { + self.auto_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid auto config", .{self.name}); + } + + /// One field per argument: `.{ .x = .{ .dtype = .bf16, .axes = &.{ M, K } } }`. + /// With `.dram = Dram{...}` (or `.{}`) the argument is DRAM-mapped and the + /// field is its `as_logical` view. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + std.debug.assert(self.entry == null); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const scratch = self.arena.allocator(); + var types: [fields.len]*const mlir.Type = undefined; + var locs: [fields.len]*const mlir.Location = undefined; + const args = try scratch.alloc(Tensor, fields.len); + inline for (fields, 0..) |f, i| { + const s = @field(spec, f); + const axes = self.dupeAxes(s.axes); + const dram: ?Dram = if (@hasField(@TypeOf(s), "dram")) toDram(s.dram) else null; + types[i] = if (dram) |d| self.mappedType(s.dtype, axes, d) else self.logicalType(s.dtype, axes); + locs[i] = self.loc(); + args[i] = .{ .inner = undefined, .dtype = s.dtype, .axes = axes, .shards = if (dram) |d| d.shards() else 1 }; + } + const entry = mlir.Block.init(&types, &locs); + // In place from the start: the kernel verifier walks to its inputs' + // regions. `finish` sets the function type. + const func_op = func.func(self.ctx, .{ .name = self.name, .block = entry, .results = &.{}, .location = self.loc(), .visibility = null, .verify = false }); + _ = func_op.appendTo(self.module.body()); + self.func_op = func_op; + self.entry = entry; + self.block_stack.push(self.arena.allocator(), entry); + var named: ArgsOf(@TypeOf(spec)) = undefined; + inline for (fields, 0..) |f, i| { + args[i].inner = entry.argument(i); + if (@hasField(@TypeOf(@field(spec, f)), "dram")) + args[i].inner = try self.graphOp(.as_logical, &.{entry.argument(i)}, self.logicalType(args[i].dtype, args[i].axes), &.{}, null); + @field(named, f) = args[i]; + } + self.args = args; + return named; + } + + fn toDram(d: anytype) Dram { + const D = @TypeOf(d); + if (D == Dram) return d; + return .{ + .chip = if (@hasField(D, "chip")) d.chip else &.{.broadcast}, + .inner = if (@hasField(D, "inner")) d.inner else null, + .original = if (@hasField(D, "original")) d.original else &.{}, + }; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".field_names; + comptime var names: [in.len][]const u8 = undefined; + inline for (in, 0..) |f, i| names[i] = f; + return @Struct(.auto, null, &names, &@splat(Tensor), &@splat(.{})); + } + + /// Opens a `tk.(...)` tensor operation; `commit` closes it. + pub fn tensorOperation(self: *Builder, opts: TensorOperation.Options) *TensorOperation { + _ = self.entryBlock(); + const op = self.arena.allocator().create(TensorOperation) catch @panic("OOM"); + op.* = .{ .b = self, .tactic = opts.tactic, .context = opts.context, .body = mlir.Block.init(&.{}, &.{}) }; + return op; + } + + pub const GraphOptions = struct { context: ?Context = null }; + + /// `reshape(t)` to `to`, same element count. + pub fn reshape(self: *Builder, t: Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + if (elements(t.axes) != elements(to)) std.debug.panic("tcl: reshape changes the element count", .{}); + return self.graph(.reshape, &.{t}, t.dtype, to, &.{}, opts.context); + } + + /// `transmute(t)`: the same bits as `dtype` over `to`. + pub fn transmute(self: *Builder, t: Tensor, dtype: DType, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.transmute, &.{t}, dtype, to, &.{}, opts.context); + } + + /// `concat([inputs])` along the one axis of `to` the inputs lack. + pub fn concat(self: *Builder, inputs: []const Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.concat, inputs, inputs[0].dtype, to, &.{}, opts.context); + } + + /// `slice(t, axis, offset)`: the window of `to` starting at `offset` along `axis`. + pub fn slice(self: *Builder, t: Tensor, along: Axis, offset: i64, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.slice, &.{t}, t.dtype, to, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + .named(self.ctx, "offset", .int(self.ctx, .i64, offset)), + }, opts.context); + } + + pub const GatherOptions = struct { + batch_axis: ?Axis = null, + valid_length: ?Tensor = null, + context: ?Context = null, + }; + + /// `gather(table, indices, axis)`: rows of `table` along `axis`. + pub fn gather(self: *Builder, table: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: GatherOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ table, indices, opts.valid_length orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.valid_length != null); + return self.graph(.gather, inputs[0..n_inputs], table.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ScatterOptions = struct { + /// The tensor scattered into; zeros otherwise. + init: ?Tensor = null, + batch_axis: ?Axis = null, + context: ?Context = null, + }; + + /// `scatter(updates, indices, axis, init=...)`: `updates` written at + /// `indices` along `axis` of the result. + pub fn scatter(self: *Builder, updates: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: ScatterOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ updates, indices, opts.init orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.init != null); + return self.graph(.scatter, inputs[0..n_inputs], updates.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ArangeOptions = struct { + start: ?i64 = null, + step: ?i64 = null, + context: ?Context = null, + }; + + /// `arange(start=, end, step=)` over the one axis of `to`. + pub fn arange(self: *Builder, end: i64, dtype: DType, to: []const Axis, opts: ArangeOptions) FinishError!Tensor { + var options: [3]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (opts.start) |v| { + options[n] = .named(self.ctx, "start", .int(self.ctx, .i64, v)); + n += 1; + } + options[n] = .named(self.ctx, "end", .int(self.ctx, .i64, end)); + n += 1; + if (opts.step) |v| { + options[n] = .named(self.ctx, "step", .int(self.ctx, .i64, v)); + n += 1; + } + return self.graph(.arange, &.{}, dtype, to, options[0..n], opts.context); + } + + pub const VectorOptions = struct { + /// Repeats `values` cyclically along this axis. + repeat_to: ?Axis = null, + context: ?Context = null, + }; + + /// `vector([values], repeat_to=)`: a constant tensor. + pub fn vector(self: *Builder, values: []const f64, dtype: DType, to: []const Axis, opts: VectorOptions) FinishError!Tensor { + const attrs = try self.arena.allocator().alloc(*const mlir.Attribute, values.len); + for (attrs, values) |*dst, v| dst.* = if (dtype.isFloat()) .float(self.ctx, .f64, v) else .int(self.ctx, .i64, @as(i64, @intFromFloat(v))); + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "values", .array(self.ctx, attrs)), undefined }; + if (opts.repeat_to) |a| options[1] = .named(self.ctx, "repeat_to", self.axisAttr(a)); + return self.graph(.vector, &.{}, dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.repeat_to != null)], opts.context); + } + + /// `all_gather(t, axis)` across chips. + pub fn allGather(self: *Builder, t: Tensor, along: Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.all_gather, &.{t}, t.dtype, t.axes, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + }, opts.context); + } + + /// `as_dram(t)`: `t` in the DRAM mapping `dram`, to return it as such. + pub fn asDram(self: *Builder, t: Tensor, dram: Dram) FinishError!Tensor { + const out = try self.graphOp(.as_dram, &.{t.inner}, self.mappedType(t.dtype, t.axes, dram), &.{}, null); + return .{ .inner = out, .dtype = t.dtype, .axes = t.axes, .shards = dram.shards() }; + } + + /// Any `tcl.graph.*` operation; `options` are its TCL keyword fields. + pub fn graph(self: *Builder, comptime op: tcl.GraphOp, inputs: []const Tensor, dtype: DType, to: []const Axis, options: []const mlir.NamedAttribute, context: ?Context) FinishError!Tensor { + const values = try self.arena.allocator().alloc(*const mlir.Value, inputs.len); + for (values, inputs) |*v, t| v.* = t.inner; + const axes = self.dupeAxes(to); + const out = try self.graphOp(op, values, self.logicalType(dtype, axes), options, context); + return .{ .inner = out, .dtype = dtype, .axes = axes }; + } + + fn graphOp(self: *Builder, comptime op: tcl.GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, options: []const mlir.NamedAttribute, context: ?Context) FinishError!*const mlir.Value { + const g = tcl.graph(self.ctx, op, inputs, output, .{ .options = options, .context = if (context) |c| self.contextAttr(c) else null }, self.loc()); + try self.append(g); + return g.result(0); + } + + /// `for iv in 0..limit (acc = init, ...):`, closed by `yield`. + pub fn ForScope(comptime N: usize) type { + return struct { + b: *Builder, + op: *mlir.Operation, + body: *mlir.Block, + /// The loop index, a scalar i32 tensor for `indexRead`, + /// `indexWrite` and `symExpr`. + iv: Tensor, + carried: [N]Tensor, + results: [N]Tensor = undefined, + + const Self = @This(); + + /// `yield values;` closes the loop and sets `results`. + pub fn yield(self: *Self, values: [N]Tensor) FinishError!void { + const b = self.b; + var inner: [N]*const mlir.Value = undefined; + for (&inner, values) |*v, t| v.* = t.inner; + _ = tcl.yield(b.ctx, &inner, b.loc()).appendTo(self.body); + b.popBlock(); + if (!self.op.verify()) { + std.log.err("tcl: {s}: invalid loop:\n{f}", .{ b.name, self.op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + for (&self.results, self.carried, 0..) |*r, c, i| r.* = .{ .inner = self.op.result(i), .dtype = c.dtype, .axes = c.axes, .shards = c.shards }; + } + }; + } + + /// Opens a device loop from 0 to `limit`: an `Axis`, an integer, or a + /// scalar i32 `Tensor` computed in the function (`reduceMaxI32`, + /// `symExpr`). The body must index a tensor with `iv`, which bounds the + /// trip count. `inits` is a tuple of the accumulators' initial tensors. + pub fn openFor(self: *Builder, limit: anytype, inits: anytype) ForScope(@typeInfo(@TypeOf(inits)).@"struct".field_names.len) { + const N = @typeInfo(@TypeOf(inits)).@"struct".field_names.len; + var types: [N + 1]*const mlir.Type = undefined; + var locs: [N + 1]*const mlir.Location = @splat(self.loc()); + var values: [N]*const mlir.Value = undefined; + var carried: [N]Tensor = undefined; + types[0] = self.logicalType(.i32, &.{}); + inline for (0..N) |i| { + carried[i] = inits[i]; + values[i] = inits[i].inner; + types[i + 1] = inits[i].inner.type_(); + } + const body = mlir.Block.init(&types, &locs); + const tensor_limit: ?*const mlir.Value, const bound: ?*const mlir.Attribute = switch (@TypeOf(limit)) { + Tensor => .{ limit.inner, null }, + Axis => .{ null, self.axisAttr(limit) }, + else => .{ null, .int(self.ctx, .i64, limit) }, + }; + // Attached before its body is built: the index operations' verifiers + // look for their loop. + const op = tcl.loop(self.ctx, &values, types[1..], body, .{ .limit = tensor_limit, .bound = bound }, self.loc()); + _ = op.appendTo(self.currentBlock()); + self.pushBlock(body); + for (&carried, 0..) |*c, i| c.inner = body.argument(i + 1); + return .{ .b = self, .op = op, .body = body, .iv = .{ .inner = body.argument(0), .dtype = .i32, .axes = &.{} }, .carried = carried }; + } + + /// `t[i]`: the slice of `t`'s outermost axis at loop index `i`. + pub fn indexRead(self: *Builder, t: Tensor, i: Tensor) FinishError!Tensor { + return self.graph(.index_read, &.{ t, i }, t.dtype, t.axes[1..], &.{}, null); + } + + /// `acc[i] = v`, in place: `acc` must have no other use; use the result. + pub fn indexWrite(self: *Builder, acc: Tensor, i: Tensor, v: Tensor) FinishError!Tensor { + return self.graph(.index_write, &.{ acc, i, v }, acc.dtype, acc.axes, &.{}, null); + } + + /// `scratchpad()`: an uninitialized tensor, typically a loop's output that + /// `indexWrite` fills. + pub fn scratchpad(self: *Builder, dtype: DType, to: []const Axis) FinishError!Tensor { + return self.graph(.scratchpad, &.{}, dtype, to, &.{}, null); + } + + /// `full(value)`: a constant tensor, like `tcl.full`. + pub fn full(self: *Builder, value: f64, dtype: DType, to: []const Axis) FinishError!Tensor { + const v: *const mlir.Attribute = if (dtype.isFloat()) .float(self.ctx, .f64, value) else .int(self.ctx, .i64, @as(i64, @intFromFloat(value))); + return self.graph(.full, &.{}, dtype, to, &.{.named(self.ctx, "value", v)}, null); + } + + /// `reduce_max_i32(t)`: the maximum of an i32 tensor, as a scalar. + pub fn reduceMaxI32(self: *Builder, t: Tensor) FinishError!Tensor { + return self.graph(.reduce_max_i32, &.{t}, .i32, &.{}, &.{}, null); + } + + /// `sym_expr(lhs rhs)`: scalar i32 arithmetic on the device, over + /// scalar tensors (a loop index, `reduceMaxI32`) and integers. + pub fn symExpr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) FinishError!Tensor { + var inputs: [2]Tensor = undefined; + var n: usize = 0; + const l = self.symOperand(lhs, &inputs, &n); + const r = self.symOperand(rhs, &inputs, &n); + const e = tcl.exprAttr(self.ctx, @tagName(kind), &.{ l, r }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(kind) }); + return self.graph(.sym_expr, inputs[0..n], .i32, &.{}, &.{.named(self.ctx, "expr", e)}, null); + } + + fn symOperand(self: *Builder, x: anytype, inputs: *[2]Tensor, n: *usize) *const mlir.Attribute { + if (@TypeOf(x) != Tensor) return .int(self.ctx, .i64, x); + inputs[n.*] = x; + n.* += 1; + return tcl.exprAttr(self.ctx, "arg", &.{.int(self.ctx, .i64, @as(i64, @intCast(n.* - 1)))}) catch unreachable; + } + + /// `copy(src=value, dst=argument)`: `value` written back into the + /// DRAM-mapped argument `index` in place. Return the result like any + /// other: the caller sees it as an output aliasing that argument. + pub fn update(self: *Builder, index: usize, value: Tensor) FinishError!Tensor { + const arg = self.entryBlock().argument(index); + const back = try self.graphOp(.as_dram, &.{value.inner}, arg.type_(), &.{}, null); + try self.append(tcl.copy(self.ctx, back, arg, self.loc())); + self.updated.append(self.arena.allocator(), index) catch @panic("OOM"); + return .{ .inner = arg, .dtype = value.dtype, .axes = value.axes, .shards = self.args[index].shards }; + } + + /// The argument each result updates in place, or null for results the + /// function returns. + pub fn updatedArgument(self: *const Builder, result: Tensor) ?usize { + for (self.updated.items) |i| if (self.entryBlock().argument(i) == result.inner) return i; + return null; + } + + /// `return results...`. + pub fn ret(self: *Builder, results: []const Tensor) void { + self.results = self.arena.allocator().dupe(Tensor, results) catch @panic("OOM"); + } + + /// The verified module: one `func.func` carrying its axes in `tcl.axes`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + const ctx = self.ctx; + const f = self.func_op orelse return error.InvalidMlir; + const results = self.results orelse return error.InvalidMlir; + const scratch = self.arena.allocator(); + var values: std.ArrayList(*const mlir.Value) = .empty; + var result_types: std.ArrayList(*const mlir.Type) = .empty; + const updated_inputs = try scratch.alloc(i64, results.len); + for (updated_inputs, results) |*u, r| { + if (self.updatedArgument(r)) |i| { + u.* = @intCast(i); + continue; + } + u.* = -1; + try values.append(scratch, r.inner); + try result_types.append(scratch, r.inner.type_()); + } + _ = func.returns(ctx, values.items, self.loc()).appendTo(self.entry.?); + + const entry = self.entry.?; + const arg_types = try scratch.alloc(*const mlir.Type, entry.numArguments()); + for (arg_types, 0..) |*t, i| t.* = entry.argument(i).type_(); + f.setAttributeByName("function_type", .typeAttr(.function(ctx, arg_types, result_types.items))); + + const axes = try scratch.alloc(mlir.NamedAttribute, self.axes.items.len); + for (axes, self.axes.items) |*dst, d| { + dst.* = .named(ctx, d.axis.name, if (d.def) |e| self.exprAttr(e) else .int(ctx, .i64, d.axis.size)); + } + tcl.annotateFunction(ctx, f, axes, .{ + .updated_inputs = if (self.updated.items.len > 0) updated_inputs else null, + .compiler_config = self.compiler_config, + .auto_config = self.auto_config, + .fusible = self.fusible, + }); + try common.verify(self.module, "dsl/tcl"); + + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + try printed.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); + } + + fn append(self: *Builder, op: *mlir.Operation) FinishError!void { + _ = op.appendTo(self.currentBlock()); + if (!op.verify()) { + std.log.err("tcl: {s}: invalid operation:\n{f}", .{ self.name, op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + } + + fn entryBlock(self: *const Builder) *mlir.Block { + return self.entry orelse std.debug.panic("tcl: {s}: declareArgs first", .{self.name}); + } + + pub fn pushBlock(self: *Builder, block: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), block); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *const Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn axisAttr(self: *const Builder, a: Axis) *const mlir.Attribute { + return tcl.symbolAttr(self.ctx, a.name) catch unreachable; + } + + fn axisAttrs(self: *Builder, axes: []const Axis) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, axes.len) catch @panic("OOM"); + for (attrs, axes) |*dst, a| dst.* = self.axisAttr(a); + return attrs; + } + + fn axesAttr(self: *Builder, axes: []const Axis) *const mlir.Attribute { + return .array(self.ctx, self.axisAttrs(axes)); + } + + pub fn exprAttr(self: *Builder, e: Expr) *const mlir.Attribute { + return switch (e) { + .int => |n| .int(self.ctx, .i64, n), + .axis => |a| self.axisAttr(a), + .broadcast => tcl.exprAttr(self.ctx, "broadcast", &.{}) catch unreachable, + .node => |n| tcl.exprAttr(self.ctx, @tagName(n.kind), &.{ self.exprAttr(n.lhs), self.exprAttr(n.rhs) }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(n.kind) }), + }; + } + + fn exprAttrs(self: *Builder, exprs: []const Expr) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, exprs.len) catch @panic("OOM"); + for (attrs, exprs) |*dst, e| dst.* = self.exprAttr(e); + return attrs; + } + + pub fn contextAttr(self: *Builder, c: Context) *const mlir.Attribute { + const ctx = self.ctx; + var fields: [2]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (c.layout) |l| { + var layout: [3]mlir.NamedAttribute = undefined; + var m: usize = 0; + layout[m] = .named(ctx, "Chip", self.exprAttr(l.chip)); + m += 1; + if (l.cluster) |cl| { + layout[m] = .named(ctx, "Cluster", self.exprAttr(cl)); + m += 1; + } + if (l.split.len > 0) { + layout[m] = .named(ctx, "Split", self.axesAttr(l.split)); + m += 1; + } + fields[n] = .named(ctx, "operator", .dict(ctx, layout[0..m])); + n += 1; + } + if (c.heuristic_hint.len > 0) { + const hints = self.arena.allocator().alloc(mlir.NamedAttribute, c.heuristic_hint.len) catch @panic("OOM"); + for (hints, c.heuristic_hint) |*dst, h| dst.* = .named(ctx, h.name, switch (h.value) { + .int => |v| .int(ctx, .i64, v), + .float => |v| .float(ctx, .f64, v), + .boolean => |v| .boolean(ctx, v), + .axis => |a| self.axisAttr(a), + }); + fields[n] = .named(ctx, "heuristic_hint", .dict(ctx, hints)); + n += 1; + } + return tcl.contextAttr(ctx, fields[0..n]) catch std.debug.panic("tcl: {s}: invalid context", .{self.name}); + } + + fn jsonFields(self: *Builder, config: anytype) []const mlir.NamedAttribute { + const fields = @typeInfo(@TypeOf(config)).@"struct".field_names; + const out = self.arena.allocator().alloc(mlir.NamedAttribute, fields.len) catch @panic("OOM"); + inline for (fields, 0..) |f, i| out[i] = .named(self.ctx, f, self.json(@field(config, f))); + return out; + } + + fn json(self: *Builder, v: anytype) *const mlir.Attribute { + const T = @TypeOf(v); + return switch (@typeInfo(T)) { + .bool => .boolean(self.ctx, v), + .comptime_int, .int => .int(self.ctx, .i64, v), + .comptime_float, .float => .float(self.ctx, .f64, v), + .enum_literal, .@"enum" => .string(self.ctx, @tagName(v)), + .@"struct" => |info| if (info.is_tuple) blk: { + var items: [info.field_names.len]*const mlir.Attribute = undefined; + inline for (info.field_names, 0..) |f, i| items[i] = self.json(@field(v, f)); + break :blk .array(self.ctx, &items); + } else .dict(self.ctx, self.jsonFields(v)), + .pointer => if (std.meta.Elem(T) == u8) .string(self.ctx, v) else blk: { + const items = self.arena.allocator().alloc(*const mlir.Attribute, v.len) catch @panic("OOM"); + for (items, v) |*dst, x| dst.* = self.json(x); + break :blk .array(self.ctx, items); + }, + else => @compileError("tcl: config values are JSON: bool, numbers, strings, structs, tuples"), + }; + } + + fn mappedType(self: *Builder, dtype: DType, axes: []const Axis, dram: Dram) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + const inner = if (dram.inner) |e| self.exprAttrs(e) else blk: { + var rest: std.ArrayList(Axis) = .empty; + for (axes) |a| if (!dram.isChipAxis(a)) rest.append(self.arena.allocator(), a) catch @panic("OOM"); + break :blk self.axisAttrs(rest.items); + }; + const mapping = tcl.dramMappingAttr(self.ctx, self.exprAttrs(dram.chip), inner, self.axisAttrs(dram.original)) catch + std.debug.panic("tcl: {s}: invalid DRAM mapping", .{self.name}); + return tcl.mappedType(self.ctx, dtype.toMlir(self.ctx), mapping) catch unreachable; + } + + pub fn logicalType(self: *Builder, dtype: DType, axes: []const Axis) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + return tcl.logicalType(self.ctx, dtype.toMlir(self.ctx), self.axisAttrs(axes)) catch + std.debug.panic("tcl: {s}: invalid tensor type (repeated axis?)", .{self.name}); + } + + fn dupeAxes(self: *Builder, axes: []const Axis) []const Axis { + return self.arena.allocator().dupe(Axis, axes) catch @panic("OOM"); + } + + fn unionAxes(self: *Builder, a: []const Axis, b: []const Axis) []const Axis { + var out: std.ArrayList(Axis) = .empty; + const long, const short = if (a.len >= b.len) .{ a, b } else .{ b, a }; + out.appendSlice(self.arena.allocator(), long) catch @panic("OOM"); + for (short) |x| if (!containsAxis(out.items, x)) out.append(self.arena.allocator(), x) catch @panic("OOM"); + return out.items; + } +}; + +fn elements(axes: []const Axis) i64 { + var n: i64 = 1; + for (axes) |a| n *= a.size; + return n; +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "softmax matches furiosa.tcl.examples.softmax" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "softmax_kernel"); + defer b.deinit(); + + const B = b.axis("B", 2); + const S = b.axis("S", 16); + const D = b.axis("D", 1024); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const centered = x.subf(x.reduce(&.{D}, .maxf)); + const exped = centered.exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tcl.axes = {B = 2 : i64, D = 1024 : i64, S = 16 : i64}"); + try expectContains(ir, "options = #tcl.read_options<{typecast_to = f32}>"); + try expectContains(ir, "mode = #tcl.reduce_mode<\"Maxf\">"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"divf\">"); + try expectContains(ir, "tactic = #tcl.tactic<\"ReduceByVe\">"); + try expectContains(ir, "-> !tcl.logical, #tcl.symbol<\"S\">, #tcl.symbol<\"D\">]>"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "batched matmul, scalar operands and graph reshape" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "bmm_scaled"); + defer b.deinit(); + + const B = b.axis("B", 2); + const M = b.axis("M", 128); + const K = b.axis("K", 64); + const N = b.axis("N", 256); + const t = try b.declareArgs(.{ + .lhs = .{ .dtype = .bf16, .axes = &.{ B, M, K } }, + .rhs = .{ .dtype = .bf16, .axes = &.{ B, K, N } }, + }); + + const mm = b.tensorOperation(.{}); + const acc = mm.contract(t.lhs, t.rhs, &.{ B, M, N }); + const prod = try mm.commit(acc.mulf(0.5), .{ .dtype = .bf16 }); + const flat = try b.reshape(prod, &.{ B, b.axis("MN", 128 * 256) }, .{}); + b.ret(&.{ prod, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); + try expectContains(ir, "\"tcl.dpe\""); + try expectContains(ir, "arith.constant 5.000000e-01 : f32"); + try expectContains(ir, "\"tcl.graph.reshape\""); +} + +test "conv1d matches furiosa.kernels.convolution" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + + const N = b.axis("N", 2); + const C = b.axis("C", 64); + const L = b.axis("L", 128); + const K = b.axis("K", 128); + const Lw = b.axis("Lw", 3); + const Lf = b.axis("Lf", 128); + const t = try b.declareArgs(.{ + .inp = .{ .dtype = .bf16, .axes = &.{ N, C, L } }, + .flt = .{ .dtype = .bf16, .axes = &.{ K, C, Lw } }, + .bias = .{ .dtype = .bf16, .axes = &.{K} }, + }); + + const op = b.tensorOperation(.{}); + const windows = op.fetch(t.inp, .{ + .pad = &.{.{ .axis = L, .left = 1, .right = 1 }}, + .slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}, + }); + try std.testing.expectEqualStrings("Lw", windows.axes[2].name); + const out = try op.commit(op.contract(windows, t.flt, &.{ N, K, Lf }).addf(t.bias), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "slide = {L = {dilation = 1 : i64, frame_axis = #tcl.symbol<\"Lf\">, stride = 1 : i64, undilated_window = 3 : i64, window_axis = #tcl.symbol<\"Lw\">}}"); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); +} + +test "composite axes, DRAM, fixed point, context, config and graph ops" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "features"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const MN = b.axis("MN", b.expr(.mul, M, N)); + try std.testing.expectEqual(64 * 128, MN.size); + b.compilerConfig(.{ .lowering_mode = "Heuristic", .max_num_partitioning_axes = 1 }); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ M, N }, .dram = .{} }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + + const op = b.tensorOperation(.{ .context = .{ + .layout = .{ .chip = .broadcast, .split = &.{M} }, + .heuristic_hint = &.{.{ .name = "max_num_partitioning_axes", .value = .{ .int = 2 } }}, + } }); + const sum = try op.commit(op.fetch(t.x, .{}).reduce(&.{N}, .addf).toFxp(15).toFp(15), .{}); + + const rows = try b.gather(t.x, t.idx, M, &.{ G, N }, .{}); + const back = try b.scatter(rows, t.idx, M, &.{ M, N }, .{ .init = t.x }); + const iota = try b.arange(N.size, .i32, &.{N}, .{}); + const pattern = try b.vector(&.{ 1, 2 }, .f32, &.{N}, .{ .repeat_to = N }); + const bits = try b.transmute(back, .i32, &.{ M, N }, .{}); + const flat = try b.asDram(try b.reshape(bits, &.{MN}, .{}), .{}); + b.ret(&.{ sum, iota, pattern, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "MN = #tcl.expr<\"mul\", [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">]>"); + try expectContains(ir, "tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\", max_num_partitioning_axes = 1 : i64}>"); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">], []>>"); + try expectContains(ir, "\"tcl.graph.as_logical\"(%arg0)"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"to_i15\">"); + try expectContains(ir, "operator = {Chip = #tcl.expr<\"broadcast\", []>, Split = [#tcl.symbol<\"M\">]}"); + inline for (.{ "gather", "scatter", "arange", "vector", "transmute", "reshape", "as_dram" }) |g| + try expectContains(ir, "\"tcl.graph." ++ g ++ "\""); +} + +test "scatter written back into its DRAM argument in place" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "write_rows"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const t = try b.declareArgs(.{ + .cache = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }, + .rows = .{ .dtype = .bf16, .axes = &.{ G, N } }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + const written = try b.update(0, try b.scatter(t.rows, t.idx, M, &.{ M, N }, .{ .init = t.cache })); + try std.testing.expectEqual(@as(?usize, 0), b.updatedArgument(written)); + b.ret(&.{written}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.copy\""); + try expectContains(ir, "tcl.updated_inputs = array"); +} + +test "loop over a device-computed trip count with index reads and writes" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "prefix"); + defer b.deinit(); + + const T = b.axis("T", 8); + const D = b.axis("D", 64); + const B = b.axis("B", 4); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ T, D } }, + .s = .{ .dtype = .f32, .axes = &.{D} }, + .lens = .{ .dtype = .i32, .axes = &.{B} }, + }); + // Running sums of the rows up to the longest length. + const n = try b.symExpr(.add, try b.reduceMaxI32(t.lens), 0); + var loop = b.openFor(n, .{ t.s, try b.scratchpad(.f32, &.{ T, D }) }); + { + const op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); + try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); + } + b.ret(&loop.results); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.for\""); + try expectContains(ir, "operandSegmentSizes = array"); + try expectContains(ir, "\"tcl.graph.index_write\""); + try expectContains(ir, "#tcl.expr<\"add\", [#tcl.expr<\"arg\", [0]>, 0]>"); +} + +test "multi-chip kernel over chip-split DRAM tensors" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "split_exp"); + defer b.deinit(); + + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = Dram{ .chip = &.{.{ .axis = C }} } } }); + try std.testing.expectEqual(2, t.x.shards); + + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"A\">], []>>"); + try expectContains(ir, "operator = {Chip = #tcl.symbol<\"C\">}"); + try expectContains(ir, "\"tcl.graph.all_gather\""); +} diff --git a/dsl/tcl/dtype.zig b/dsl/tcl/dtype.zig new file mode 100644 index 00000000..9faf1e18 --- /dev/null +++ b/dsl/tcl/dtype.zig @@ -0,0 +1,57 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// TCL element types, named as in TCL (`[A]/f8_e4`). +pub const DType = enum { + bool, + i4, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f16, + bf16, + f32, + f64, + f4_e2, + f8_e4, + f8_e5, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f4_e2 => .float(ctx, .f4e2m1fn), + .f8_e4 => .float(ctx, .f8e4m3fn), + .f8_e5 => .float(ctx, .f8e5m2), + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f4_e2, .f8_e4, .f8_e5 => true, + else => false, + }; + } +}; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/triton/BUILD.bazel b/dsl/triton/BUILD.bazel new file mode 100644 index 00000000..6699694a --- /dev/null +++ b/dsl/triton/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/triton/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/scf", + "//mlir/dialects/ttir", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/triton/PORTING.md b/dsl/triton/PORTING.md new file mode 100644 index 00000000..0d328710 --- /dev/null +++ b/dsl/triton/PORTING.md @@ -0,0 +1,758 @@ +# Porting Triton kernels from Python to the Zig DSL + +This guide covers the day-to-day workflow for **adding a new Triton kernel +to ZML** (port from a Python `@triton.jit` source) and **fixing an existing +Zig kernel that diverges from Python** (using the comparison harness in +`examples/triton_emitter/`). + +> **The harness is not in this repository.** `examples/triton_emitter/` and +> its scripts (`run.sh`, `dump_python_ir.py`, `kernels_zig.zig`, +> `dump_via_xla.zig`, `compare_ir.py`) are not part of ZML; the steps that +> use them describe the comparison it runs. Writing the kernel and its +> caller (Steps 1.2 and 1.3) and the porting bugs of Phase 3 don't need it. + +## Goals (in priority order) + +1. **Lowered code matches.** The Zig version produces **the same LLVM IR + + PTX** as the Python version. TTIR/TTGIR may differ cosmetically (op + ordering, naming) — what matters is what actually runs on the GPU. + +2. **Source code looks alike.** A Python kernel and its Zig port should be + *visually transparent* side by side: same overall structure, same + variable names, same comment placement, same line ordering. Someone + reading both should be able to follow them in lockstep. + + Visual fidelity matters because: + - It makes future ports trivial — patterns transfer one-to-one. + - It makes upstream-Triton patches easy to apply (the Python diff + translates almost mechanically to a Zig diff). + - It catches porting bugs before they reach the comparison harness: + when the Zig code reads differently from Python, that's where the + real-vs-cosmetic divergence usually hides. + + Concretely: if a Python source line is + ```python + y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) * group_size) + ``` + the Zig port should be one line, named the same way: + ```zig + const y_ptr_offset = row.to(.i64).mul(y_row_stride).add(row_g_id.to(.i64).mul(group_size)); + ``` + Not split across three temporaries with renamed locals — even if the + resulting LLIR is identical. Goal #1 is a hard requirement; goal #2 + is what makes this codebase maintainable. + +For the DSL surface (`zml.kernel.triton.Kernel`, `Builder`, `Value`, etc.), see +[`README.md`](./README.md). This file is about the *process*, not the API. + +--- + +## Prerequisites + +- CUDA or ROCm GPU (XLA's Triton backend is GPU-only). +- The Python venv at `examples/triton_emitter/.venv` activated, or a + system-level Python with `triton`, `torch`, `numpy` installed. +- Bazel + the standard ZML build deps. + +One-time: + +```sh +cd examples/triton_emitter +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +--- + +## Phase 1 — Initial port + +Start by copying the Python kernel verbatim into the Zig DSL form. The +goal is a kernel that *compiles*, *verifies*, **and reads as similarly +to the Python source as possible**. + +> **Author hint:** open the Python file in a left pane and the Zig file +> in a right pane. Translate **line by line**, top to bottom. Keep +> variable names, comment placement, and intermediate-binding granularity +> identical wherever the DSL allows. Don't refactor, don't rename, don't +> "improve" — that's a separate task you do after the LLIR matches. + +### Step 1.1 — Drop the Python source next to the Zig file + +For comparison purposes, the Python source has to live under +`examples/triton_emitter/kernels_py/`. The dump script auto-discovers +every `@triton.jit` function in that directory. + +If your kernel's already in `vllm/` or another upstream tree, copy the +relevant `@triton.jit` definition (and any `@triton.jit` helpers it +calls) into a single `.py` file there. Strip away launch wrappers, +autotune decorators, and any code that isn't actually inside the kernel +function. + +### Step 1.2 — Sketch the Zig declaration + +Inside `zml/kernels/moe/triton/` or `zml/kernels/paged_attention/triton/` +(or whichever production file fits), add: + +```zig +const tri = zml.kernel.triton; + +pub const MyKernel = struct { + pub const Cfg = struct { + // One field per `tl.constexpr` parameter, named as in Python. + BLOCK_SIZE: usize, + // ... + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` + // Call-site names, in `tt.func` argument order: inputs, then outputs. + .inputs = &.{"x_ptr"}, + .outputs = &.{"out_ptr"}, + .run = run, + }); + + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + // One field per *runtime* parameter (pointers, dims that + // arrive as 0-d tensors). Use the same names + order Python + // uses; the README's "Argument specs" table maps the + // tagged-union variants. + .x_ptr = .{ .ptr = .f32 }, + // ... + }); + + // Body — translate Python line by line. Cheat-sheet in the + // README's "Python Triton ↔ Zig DSL cheat-sheet" section is + // your best friend here. + } +}; +``` + +### Step 1.3 — Wire the production caller + +Inside the same `.zig` directory, find the function that launches your +kernel (e.g. `pagedAttention2d`, `fusedExpertsImpl`). Replace the +TTIR-emit + `ops.triton(...)` two-step with a single +`MyKernel.Kernel.call(inputs, outputs, opts)` — see the +[README](./README.md#calling-kernels-from-a-zml-model--kkernelcall) for the +exact shape. + +If this is a brand-new kernel with no existing caller, write the caller +fresh. + +### Step 1.4 — Wire the comparison harness + +Add the kernel to `examples/triton_emitter/kernels_zig.zig`: + +```zig +withConfig(moe.MyKernel, .{ .BLOCK_SIZE = 64, /* ... */ }), +``` + +If the kernel takes runtime tensor inputs, also add a `forward` struct + +`KERNELS` entry to `examples/triton_emitter/dump_via_xla.zig` so XLA's +codegen pipeline can lower it. The tensor sizes are dummies — XLA +doesn't launch the kernel, it just runs the lowering passes. + +For the Python side, the auto-discovery in `dump_python_ir.py` works for +simple kernels (only `*_ptr` runtime args + `tl.constexpr` params). +**If the signature mixes runtime args and constexpr in the middle of the +parameter list**, write an override function and add it to `_OVERRIDES` +(follow `_per_token_group_quant_fp8` / `_kernel_unified_attention_2d_ptr` +as templates). Use the same constexpr values + tensor shapes you used in +the Zig `withConfig(...)` call. + +### Step 1.5 — Build it + +```sh +bazel build //examples/triton_emitter:dump_zig_ir # Zig side compiles + verifies +bazel build //zml:test # production side compiles +``` + +If the Zig DSL `b.finish()` rejects your IR, it logs the module. Mixed +integer widths and scalar/tensor operands are not the cause: the fluent ops +widen the narrower integer side (`v_i32.mul(v_i64)` is an i64 multiply), and +`addPtr` splats a scalar pointer or offset against a tensor partner. A common +cause is an **`scf.yield` type mismatch** — the values you pass to +`loop.yield(...)` / `i.yieldThen(...)` must have the types of the loop's +`inits` / the if's `result_types` (their arity is checked at compile time). + +### Step 1.6 — First dump + +```sh +bash examples/triton_emitter/run.sh +``` + +This runs the full pipeline (Python TTIR → XLA → per-stage IR; same on +the Zig side; diff each stage). At the end you get a per-kernel +verdict: + +``` +my_kernel + ttir +30/-30 + ttgir +30/-30 + llir +50/-50 + ptx +40/-40 +``` + +Don't worry yet about cosmetic diffs — go through Phase 2 to find +*real* divergences first. + +--- + +## Phase 2 — The comparison pipeline + +The harness has six steps (see `run.sh`), output appears under +`examples/triton_emitter/`: + +| Step | Output | What it is | +|------|------------------------------|-----------------------------------------------| +| 1 | `py_ir/*.ttir`/`.ttgir`/... | Python's per-stage IR (Triton's own pipeline) | +| 2 | `zig_ir/*.ttir` | Just the TTIR your DSL emits. | +| 3 | `xla_dump_py/` | XLA's per-pass dumps lowering Python's TTIR. | +| 4 | `xla_dump_zig/` | XLA's per-pass dumps lowering the Zig TTIR. | +| 5 | `xla_py/*.{ttir,ttgir,llir,ptx}` + `xla_zig/*.{...}` | Per-stage IR extracted by `extract_xla_dump.py`. Both sides go through XLA, not Triton-the-Python-frontend, so the comparison is apples-to-apples. | +| 6 | stdout | `compare_ir.py` diff summary per kernel. | + +### Useful flags + +```sh +# Just one kernel (faster iteration): +# (use ZIG_IR/PY_IR filter — but note dump_zig_ir's CLI expects --kernel=NAME). +bash run.sh # full run +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs + +# Skip the slow XLA-lowering steps if only the TTIR changed: +python dump_python_ir.py --out-dir py_ir +bazel run //examples/triton_emitter:dump_zig_ir -- --out-dir=$PWD/zig_ir +diff py_ir/my_kernel.ttir zig_ir/my_kernel.ttir +``` + +### Reading the verdict + +``` +my_kernel + ttir +30/-30 <- TTIR diverges 30 lines + ttgir +30/-30 <- so does TTGIR (same shape) + llir ✓ <- but LLIR converges! cosmetic-only TTIR diff + ptx ✓ <- and PTX matches +``` + +This kernel is **good** — final code is identical, the TTIR diff is just +op-naming/ordering cosmetics that XLA's lowering canonicalizes away. + +``` +my_kernel + ttir +5/-5 + ttgir +5/-5 + llir +60/-58 <- LLIR diverges → real semantic difference + ptx +50/-48 +``` + +This kernel needs work: a real divergence appeared somewhere between +TTIR and LLIR. Drill into the LLIR diff to find which op got lowered +differently. + +--- + +## Phase 3 — Fixing divergences + +Workflow: pick the *smallest* diverging stage's diff and locate the +op-by-op mismatch. Most fixes fall into one of the categories below. + +### Step 3.1 — `--show-diffs` to locate the divergence + +```sh +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs +``` + +Read the diff carefully. The first diverging op tells you what to look +at; subsequent diffs are usually downstream consequences. + +### Step 3.2 — Common porting bugs + +These are the gotchas we've hit while porting kernels. If your diff +shows any of these patterns, fix the corresponding Zig DSL call. + +#### 3.2.1 — `arith.maxnumf` vs `arith.maximumf` (and `min*`) + +**Symptom:** +``` +- %v43 = arith.maxnumf %v37, %v14 : f32 (Python) ++ %v43 = arith.maximumf %v37, %v13 : f32 (Zig) +``` + +**Fix:** **Use `b.maximum` / `Value.maximum` / `b.minimum` / +`Value.minimum`** — they default to `maxnumf`/`minnumf` and match +Python's `tl.max`/`tl.min`/`tl.maximum`/`tl.minimum`. **Don't reach +for `b.maximumf` / `b.minimumf` directly** — those are the IEEE-754 +NaN-propagating escape hatches (`arith.maximumf`/`minimumf`), exposed +for the rare case where you genuinely want NaN propagation. The naming +is unfortunate (the `f` suffix reads like "the float version") — it's +actually "the NaN-propagating version." See README's "Reductions and +scans" gotcha table for the full mapping. + +#### 3.2.2 — 2-operand vs 3-operand `tt.load` + +**Symptom:** +``` +- %v33 = tt.load %v32, %v30, %v9 : tensor<128x!tt.ptr> (Python — has `other`) ++ %v33 = tt.load %v32, %v30 : tensor<128x!tt.ptr> (Zig — no `other`) +``` + +**Fix:** Match Python's exact form. The DSL has only one masked-load +API (`b.loadOpts`); whether it emits 2-operand or 3-operand `tt.load` +depends entirely on whether you pass `.other`: + +```zig +// Python: tl.load(ptr, mask=mask, other=0.0) → 3-operand +const zero = b.zeros(&.{BLOCK}, dtype); +const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); + +// Python: tl.load(ptr, mask=mask) → 2-operand +const x = b.loadOpts(ptr, .{ .mask = mask }); +``` + +The two forms are *semantically* close but not identical (out-of-bounds +lanes are undefined for 2-operand, zero for 3-operand). Always copy +Python's form, even if you suspect the difference is harmless — XLA's +canonicalization may or may not erase it. + +#### 3.2.3 — `i32` vs `i64` width in pointer offsets + +**Symptom:** +``` +- %v26 = tt.addptr %v6, %v16 : !tt.ptr, i32 (Python — uses i32) ++ %v25 = tt.addptr %v6, %v16 : !tt.ptr, i64 (Zig — uses i64) +``` + +**Fix:** Python's pointer arithmetic preserves the original integer +width. `tl.program_id(0)` is i32; if Python writes +`y_s_ptr += tl.program_id(0)` (no cast), the offset stays i32. In Zig, +if you converted `pid` to i64 early to use it elsewhere, pass the raw +`b.programId(.x)` (i32) to that one `addPtr` site: + +```zig +// raw i32 program_id for one site, i64 elsewhere +const pid_i32 = b.programId(.x); +const g_id = pid_i32.to(.i64); +// ... +const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); // i32, not g_id +``` + +#### 3.2.4 — Mask comparison width + +**Symptom:** +``` +- %v30 = arith.cmpi slt, %v28, %v29 : tensor<128xi64> (Python — i64 vs i64) ++ %v28 = arith.trunci %v10 : i64 to i32 ++ %v29 = tt.splat %v28 : i32 -> tensor<128xi32> ++ %v30 = arith.cmpi slt, %v26, %v29 : tensor<128xi32> (Zig — trunc + splat + i32 cmp) +``` + +**Fix:** Match Python's promotion direction. Python implicitly promotes +the narrower operand to the wider one. If `cols` is i32 and +`group_size` is i64, Python emits `extsi cols i32→i64` then compares +at i64. The fluent ops promote the same way, so compare the two as they +are; a `trunci` like the one above comes from narrowing one side by hand: + +```zig +const cols = b.arange(0, BLOCK, .i32); +const mask = cols.lt(group_size); // extsi cols to i64, no trunc +``` + +#### 3.2.5 — Op emit order + +**Symptom:** TTIR diff shows the same ops in a different order: +``` +- %v18 = arith.divsi %v17, %v15 : i64 +- %v19 = arith.remsi %v17, %v15 : i64 +- %v20 = arith.muli %v18, %v13 : i64 +- %v21 = arith.muli %v19, %v11 : i64 +- %v22 = arith.addi %v20, %v21 : i64 ++ %v17 = arith.divsi %v16, %v14 : i64 ++ %v18 = arith.muli %v17, %v12 : i64 ++ %v19 = arith.remsi %v16, %v14 : i64 ++ %v20 = arith.muli %v19, %v10 : i64 ++ %v21 = arith.addi %v18, %v20 : i64 +``` + +This usually doesn't affect LLIR/PTX (compiler reorders), but if it +does (or if you want a clean TTIR diff), match Python's source order +op-by-op. Bind every intermediate to a named local in the same order +Python evaluates: + +```zig +// Python: y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size +// eval order: divsi, remsi, muli, muli, addi +const row = g_id.div(groups_per_row); // divsi +const row_g_id = g_id.rem(groups_per_row); // remsi +const row_off = row.mul(y_row_stride); // muli +const grp_off = row_g_id.mul(group_size); // muli +const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); // addi + addptr +``` + +#### 3.2.6 — `arith.maxnumf` (and `splat`) vs `tt.splat` of a runtime value + +**Symptom:** +``` +- %splat = tt.splat %scalar : i32 -> tensor<128xi32> (Python — runtime splat) ++ %const = arith.constant dense<...> : tensor<128xi32> (Zig — constant pool) +``` + +**Fix:** `b.splat(comptime_literal, shape)` materializes a constant +dense; `b.splat(runtime_value, shape)` emits `tt.splat`. If Python +uses a runtime value (loaded scalar, function arg) and you accidentally +hardcoded the constant, switch to the runtime path. + +#### 3.2.7 — `tl.constexpr` divisibility hints + +**Symptom:** TTIR header differs in `tt.divisibility = N : i32` attrs. + +**Fix:** Default `.{ .ptr = .dtype }` carries `tt.divisibility = 16`. +For scalars, switch to `.scalar_opts`: + +```zig +.n_elements = .{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }, +``` + +This matches what Python's JIT runtime auto-attaches when the runtime +value is a multiple of 16. + +#### 3.2.8 — Reductions over rank-1 tensors + +**Symptom:** Python emits a `tt.reshape allow_reorder` before the +`tt.reduce`; Zig doesn't. + +**Fix:** Use the `axis = null` (default) form: `b.max(x)`, not +`b.maxOpts(x, .{ .axis = 0 })`. The `axis = null` path inserts the +reshape; explicit-axis paths skip it. + +#### 3.2.9 — `tl.dot` precision and accumulator + +**Symptom:** TTIR shows different `tt.dot` `inputPrecision` attribute or +different `acc` type. + +**Fix:** Match Python's `tl.dot(a, b, acc, input_precision="...")` +explicitly: + +```zig +const new_acc = b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, // matches `input_precision="tf32"` + .max_num_imprecise_acc = 0, +}); +``` + +Default `input_precision` differs across Triton versions — always be +explicit. + +#### 3.2.10 — `iter_args` order in `scf.for` + +**Symptom:** TTIR diff shows the loop signature reordered: +``` +- %v152:3 = scf.for ... iter_args(%v149 = %v131, %v150 = %v143, %v151 = %v37) +- -> (tensor<...!tt.ptr>, tensor<...!tt.ptr>, tensor<...xf32>) ++ %v152:3 = scf.for ... iter_args(%v149 = %v37, %v150 = %v131, %v151 = %v143) ++ -> (tensor<...xf32>, tensor<...!tt.ptr>, tensor<...!tt.ptr>) +``` + +**Fix:** Python's `scf.for` lowering captures `iter_args` in the order +they're **declared** above the loop, not the order they're mutated inside. +For `fused_moe_kernel`: + +```python +a_ptrs = ... # declared 1st +b_ptrs = ... # declared 2nd +accumulator = tl.zeros(...) # declared 3rd +for k in range(...): + accumulator += tl.dot(a, b) # mutated 1st + a_ptrs += BLOCK_SIZE_K # mutated 2nd + b_ptrs += BLOCK_SIZE_K # mutated 3rd +``` + +Python's `iter_args` order = declaration order = `(a_ptrs, b_ptrs, acc)`, +**not** mutation order. The DSL `openFor` follows the order you pass — so +match Python's declaration order: + +```zig +var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); +{ + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + // ... + loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); +} +const accumulator = loop.results[2]; +``` + +The wrong order survives all the way to PTX as different register-block +layouts. + +#### 3.2.11 — Operand order in commutative `arith.muli` + +**Symptom:** TTIR diff shows `arith.muli` operands swapped: +``` +- %v137 = arith.muli %v136, %v135 : tensor<1x64xi64> (Python: stride on LHS) ++ %v137 = arith.muli %v135, %v136 : tensor<1x64xi64> (Zig: offs on LHS) +``` + +**Fix:** `arith.muli` is commutative semantically, but the operand order +affects downstream layout decisions in TTGIR. Match Python's source order +exactly. For `stride * offs[None, :]`, Python emits `arith.muli %stride_splat, +%offs`: + +```zig +// Python: stride_cm * offs_token[:, None] — stride on LHS +const cm_col = stride_cm.mul(offs_token_col); +// NOT: offs_token_col.mul(stride_cm) — offs on LHS, wrong order +``` + +Same applies to `arith.addi`, `arith.subi`, `arith.andi`, etc. + +#### 3.2.12 — Two-step `addPtr` for `c_ptr + cm[:, None] + cn[None, :]` + +**Symptom:** Python emits two `tt.addptr` ops; the Zig port collapses +them into one big addptr with a pre-merged 2D offset. +``` +Python: + %ptr_col = tt.addptr %c_ptr_splat_Mx1, %cm_col_Mx1 : Mx1x!tt.ptr, Mx1xi64 + %ptr_2d = tt.broadcast %ptr_col → MxNx!tt.ptr + %final_ptr = tt.addptr %ptr_2d, %cn_off_2d : MxNx!tt.ptr, MxNxi64 +Zig: + %offset = arith.addi %cm_term, %cn_term : MxNxi64 + %final_ptr = tt.addptr %c_ptr_splat, %offset : MxNx!tt.ptr, MxNxi64 +``` + +**Fix:** Python evaluates `c_ptr + cm[:, None] + cn[None, :]` left-to-right, +so it emits two `tt.addptr` ops. Match by splitting: + +```zig +// First addptr: scalar c_ptr + (Mx1) cm offset → Mx1 ptr tensor +const cm_col = stride_cm.mul(b.expandDims(offs_token, 1)); +const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + +// Second addptr: broadcast to MxN, add cn offset +const cn_row = b.expandDims(offs_cn, 0); +const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); +const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(cn_row, &.{ block_size_m, block_size_n })); +``` + +This pattern shows up wherever Python writes `ptr + a[:, None] + b[None, :]` +(both A and B ptrs in fused_moe, the C-output store, etc.). + +#### 3.2.13 — `extsi` placement relative to sibling `expand_dims`/`mul` + +**Symptom:** Same set of ops, different positions: +``` +- %v134 = expand_dims %offs_k_i32 (axis=1) (Python: offs_k expand) +- %v135 = expand_dims %offs_bn_i64 (axis=0) (Python: offs_bn expand) +- %v136 = splat %stride_bn → 1x64xi64 (Python: bn stride) +- %v137 = muli %v135, %v136 (Python: bn_row mul) +- %v138 = extsi %v134 → 32x1xi64 (Python: extsi DEFERRED) ++ %v135 = extsi %v134 → 32x1xi64 (Zig: extsi EAGER) ++ ... (rest in different order) +``` + +**Fix:** When Python writes `offs_k[:, None]` and uses it later in an +`addi` against an `i64`-typed tensor, the implicit `extsi` happens at the +**use site** (where the auto-promotion fires), not at the `expand_dims`. +Mirror this — keep the chain at i32 until just before it joins the i64 +expression: + +```zig +// Wrong: extsi happens immediately after expand_dims +const offs_k_col = b.expandDims(offs_k_i32, 1).to(.i64); // extsi NOW +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); + +// Right: extsi deferred to after offs_bn's mul +const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); // 32x1xi32 +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); // 1x64xi64 +const offs_k_col = offs_k_col_i32.to(.i64); // extsi HERE +``` + +Same idea for placing a `muli` before an `extsi` on a sibling load: + +```zig +// Python emits muli before the extsi at the cmpi: +const num_tokens_post_padded_i32 = b.load(...); // load +const pid_m_block = pid_m.mul(block_size_m); // muli FIRST +const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); // extsi +const out_of_range = pid_m_block.ge(num_tokens_post_padded); +``` + +### Step 3.3 — Iterate + +After each fix: + +```sh +bazel build //examples/triton_emitter:dump_zig_ir +bash examples/triton_emitter/run.sh +``` + +Re-read the diff. The line counts should drop, and ideally the LLIR/PTX +columns flip to `✓`. If not, drill again — there's another mismatch +hiding behind the first. + +### Step 3.4 — Don't be a perfectionist about TTIR (but do read every line) + +You only need **LLIR + PTX** to match. If TTIR/TTGIR have +5 cosmetic +lines but LLIR is byte-identical, ship it. The XLA canonicalization + +CSE passes erase a lot of low-level differences before LLIR. + +A kernel like `count_and_sort_expert_tokens_kernel` (TTIR/TTGIR +8/-8, +LLIR/PTX ✓) is a clean port. Don't keep grinding it. + +**But** — when LLIR is **not** ✓ and the diff is small (tens of lines), +do **not** label the rest "scheduling noise" without a name-normalized +diff. Most lines in a small LLIR diff are downstream SSA renumbering +caused by 1–4 high-leverage TTIR-level fixes (typically the patterns in +3.2.10–3.2.13). Run: + +```sh +diff \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_py/foo.llir) \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_zig/foo.llir) +``` + +If this normalized diff is small (~tens of lines), every remaining line +is pointing at a structural fix. The "+100 LLIR is just XLA scheduling" +intuition has been wrong every time so far — `fused_moe_kernel` went from +LLIR +100 → ✓ via four TTIR-level fixes (iter_args order, muli operand +order, two-step addPtr, extsi placement) that were all visible in the ++100-line diff once read carefully. + +### Step 3.5 — Visual-transparency pass + +Once the LLIR matches, do one pass over the Zig source with the Python +source open beside it. Check (in order): + +- **Variable names.** Python `groups_per_row` → Zig `groups_per_row`, + not `gp_row` or `gpr`. Same for arg names — `y_ptr` not `output_ptr`. +- **Line order.** Each Python line should correspond to roughly one + Zig line, in the same order. If a single Python expression became + three Zig temporaries, ask: does the DSL force the split? If not, + collapse them onto one chained `.add(...).mul(...)` line. +- **Granularity of locals.** If Python writes + `_absmax = tl.maximum(tl.max(tl.abs(y)), eps)` as a one-liner, the + Zig port should also be one line: `const absmax = b.max(b.absf(y)).maximum(eps);`. +- **Comments.** Carry over Python's comments verbatim where they apply. + If Python writes `# Map the program id to the row of X and Y it + should compute.` above its block, paste the same comment above the + Zig block. They serve as anchors for future side-by-side reads. +- **Helper functions.** If Python factors something into a + `@triton.jit` helper (e.g. `find_seq_idx`), the Zig port should have + a matching `fn findSeqIdx(b: *Builder, ...) Value` helper, not inline + the body. Same arguments, same name (snake_case → camelCase). +- **Control flow shape.** Python `for k in range(...)` → `b.openFor`, + Python `if ... return` → `b.returnIf` / `b.openReturnIf`. Don't + flatten an `if` ladder into a `select` chain (or vice versa) just + because the DSL allows it. + +If a transparency fix changes the LLIR diff, **the LLIR-match goal +wins** — back out the visual change and leave a comment explaining the +divergence. Goal #1 always beats goal #2. + +A useful sanity check: open both files side by side in your editor and +scroll both at the same rate. If the cursors stay roughly aligned, the +port reads transparently. If you have to scroll one twice as fast as +the other, there's structural drift to clean up. + +--- + +## Phase 4 — Production wiring (post-port) + +Once the LLIR matches *and* the visual-transparency pass is done: + +1. **Side-by-side review.** Pull up Python and Zig in adjacent panes + one more time. Anything that still reads differently between them + either has a justification (DSL constraint, language constraint — + leave a brief comment explaining why) or needs another pass. +2. **Remove any `// TODO match python` comments** from the kernel body. +3. **Verify the production caller passes the right cfg** — the dump + tool's config and the production caller's config can differ (the + dump tool uses small dimensions; production uses real ones). + Production behavior depends on the runtime cfg, so make sure + `makeFooConfig(...)` in the caller produces sensible values. +4. **Run the model end-to-end** if possible. The comparison harness + only validates IR equivalence — runtime correctness still needs a + real test (mismatched grid, missing strides, etc. won't show up in + the diff but will crash at execution time). + +--- + +## Cheat-sheet: when to look at which file + +| Problem | File | +|---------------------------------------------------|----------------------------------------------| +| "How do I express `tl.X(...)` in Zig?" | `dsl/triton/README.md` (Python ↔ Zig table) | +| "What does `b.declareArgs(...)` accept?" | `dsl/triton/README.md` (Argument specs) | +| "How do I call this kernel from `forward`?" | `dsl/triton/README.md` (`K.call(...)`) | +| "My kernel diverges from Python at LLIR" | This file (Phase 3) | +| "How do I add a kernel to the comparison harness?"| Phase 1 above + `examples/triton_emitter/` | +| "I want to inspect raw TTIR ops" | `mlir/dialects/ttir/ttir.zig` (escape hatch) | + +--- + +## When you're stuck + +If a divergence resists all the standard fixes: + +1. **Print both TTIRs side by side.** The visual structure usually + reveals which control-flow region or loop body is misaligned. +2. **Check if the Python kernel uses any non-`@triton.jit` helpers** — + if it imports something from `triton.language.extra` or has a custom + `@triton.jit` helper, that helper might also need a port (or a + `_SKIP` entry if it's pure-comptime). +3. **Verify the Python override args match the Zig config.** Mismatched + constexpr values (e.g. `BLOCK_SIZE=64` in Zig vs `1024` auto-default + in Python) generate completely different IR. +4. **Compare against the `zml/kernels/moe/triton/` MoE kernels** — they + went through this loop and are reasonable references for the patterns + above. + +--- + +## Before "fixing" the DSL + +Every time a divergence makes you suspect the DSL itself is buggy and you +reach for `dsl/triton/builder.zig` — **verify against the Triton sources +first** in a Triton checkout. The DSL exists to mirror what Python's +frontend emits; the source of truth is the Python trace, not what the rule +"looks like" or what feels symmetric. + +Concretely: + +- If you think `Value.cdiv` is wrong, read + `triton/python/triton/language/standard.py` for the definition of + `tl.cdiv` and compare. Don't assume the desugaring — check. +- If you think a binop dispatches to the wrong arith op + (`maxnumf` vs `maximumf`, `divsi` vs `divui`, …), read + `triton/python/triton/language/semantic.py` and find the cast / + promotion rule that actually fires. +- If you think the type-promotion is wrong (i32 stays i32, doesn't + auto-extend to i64, …), read `semantic.py:cast()` and the binop + promotion helpers. + +A second pitfall: **the local kernel may define its own helper that +shadows the stdlib name.** `unified_attention.py` defines a private +`cdiv_fn(x, y) = (x + y - 1) // y` — left-to-right Python parses this +as `((x + y) - 1)`, so it emits `addi` then `subi`. Triton's *stdlib* +`tl.cdiv` is `(x + (div - 1)) // div` and emits `subi` then `addi`. +These desugar identically when the divisor is comptime (Triton folds +`div - 1` at trace time), but differ when the divisor is a runtime +tensor. If your divergence is just an `add↔sub` reorder, check whether +the Python kernel actually uses `tl.cdiv` or a private `cdiv`-shaped +helper, and either keep the DSL aligned with `tl.cdiv` and write a +local Zig helper for the kernel-private form, or the other way around. + +In short: when a "bug" in the DSL would also affect kernels that +already match Python, the bug is almost certainly elsewhere. + +**And if you do change the DSL**, run `examples/triton_emitter/run.sh` +afterwards and confirm no kernel that previously matched (or was +codegen-equivalent) regressed at LLIR/PTX. A DSL edit that fixes one +kernel by introducing a divergence in another is not a fix — it's a +trade. Always check the full table before committing the change. diff --git a/dsl/triton/README.md b/dsl/triton/README.md new file mode 100644 index 00000000..939a1b07 --- /dev/null +++ b/dsl/triton/README.md @@ -0,0 +1,1116 @@ +# ZML Triton DSL — Author's Guide + +A Zig builder for Triton IR (TTIR) that tries to read like `triton.language` +Python while staying in pure Zig. Used by `zml/kernels/moe/triton/`, +`zml/kernels/paged_attention/triton/`, and by anything else that wants to call +Triton kernels from a ZML model. + +Two layers define the surface: + +- **Low layer** — `dsl/triton/builder.zig`: the IR builder primitives. + `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need + full control (custom op insertion, dynamic-arity arg lists, escape hatches). + `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a + `zml.kernel.triton.newContext()` (a throwaway context with the dialects the + DSL emits), `defer ctx.deinit()` once you have the IR string. Not a + `zml.Compiler`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` + (a registered `llvm` dialect makes XLA:TPU choke). +- **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative + kernel form. Bundles a config type, a typed spec literal (name + named + `inputs` / `outputs` + a typed `run` function pointer), and produces a + kernel type. Exposes `.emit(allocator, cfg)` (TTIR string; manages its own + throwaway context) and `.call(inputs, outputs, opts)` (TTIR string + + `stablehlo.custom_call` in one shot — the preferred form for production + model code). Inputs and outputs are by-name structs generated from the + spec, so missing / typo'd / extra arguments are compile errors at the callsite. + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [`zml.kernel.triton.Kernel(Config, spec)` — the declarative form](#zmlkerneltritonkernelconfig-spec--the-declarative-form) +3. [Calling kernels from a ZML model — `K.Kernel.call(...)`](#calling-kernels-from-a-zml-model--kkernelcall) +4. [Argument specs](#argument-specs) +5. [The `Value` type](#the-value-type) +6. [Fluent methods on `Value`](#fluent-methods-on-value) +7. [Polymorphic scalars](#polymorphic-scalars) +8. [Comptime vs runtime widths](#comptime-vs-runtime-widths) +9. [Shape helpers](#shape-helpers) +10. [Loads and stores](#loads-and-stores) +11. [Reductions and scans](#reductions-and-scans) +12. [Control flow](#control-flow) +13. [Escape hatches](#escape-hatches) +14. [Lower layer — `Builder.open`](#lower-layer--builderopen) +15. [Python Triton ↔ Zig DSL cheat-sheet](#python-triton--zig-dsl-cheat-sheet) +16. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const tri = @import("zml").kernel.triton; +const zml = @import("zml"); + +pub const MyKernel = struct { + pub const Cfg = struct { BLOCK: i32 = 64 }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x_ptr", "y_ptr", "n" }, + .outputs = &.{"out"}, + .run = run, + }); + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .x_ptr = .{ .ptr = .f32 }, + .y_ptr = .{ .ptr = .f32 }, + .n = .{ .scalar = .i32 }, + .out_ptr = .{ .ptr = .f32 }, + }); + + const pid = b.programId(.x); + const offs = b.arange(0, cfg.BLOCK, .i32).add(pid.mul(cfg.BLOCK)); + const mask = offs.lt(a.n); + + const x = b.loadOpts(a.x_ptr.addPtr(offs), .{ .mask = mask }); + const y = x.mul(2.0); + b.storeOpts(a.out_ptr.addPtr(offs), y, .{ .mask = mask }); + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. Use field defaults for sensible defaults. +2. **`pub const Kernel = tri.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body. First call `try b.declareArgs(.{…})` + to create the `tt.func` op and bind a struct of `Value`s named after the + spec's fields. The rest of `run` builds the body via `b.*` helpers. No + string return, no allocator, no manual `tt.return` — the factory + terminates, verifies, and serializes for you. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgs(...)` (which are the **TTIR-level** arg names). +By convention they often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgs(...)` is a struct literal — its variant +*tags* (`.ptr`, `.scalar`, etc.) are comptime-known per call site, but the +*payload values* (`cfg.input_dtype`, `cfg.output_dtype`, …) can be runtime +values. This is what makes the same form work for both comptime-pinned +configs (offline tools, simple kernels) and runtime configs (production +models where dtypes/sizes come from `Tensor.dim()` / `Options`). + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). + +### Two ways to use a kernel + +**Offline / tooling** — get the TTIR string (no MLIR context needed; `emit` +spins up and tears down its own): + +```zig +const ttir = try MyKernel.Kernel.emit(allocator, .{ .BLOCK = 1024 }); +defer allocator.free(ttir); +``` + +**Inside a ZML `forward(...)`** — emit TTIR and drop a `stablehlo.custom_call` +into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x_ptr = x_tensor, .y_ptr = y_tensor, .n = n_tensor }, // Inputs + .{ .out = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, + .grid = .{ ceilDiv(n, 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, +).out; +``` + +`Kernel.call` returns a `Results` struct (one `Tensor` per output name); +read fields by name (`out.out`, `align_outs.sorted_token_ids`, …). + +--- + +## `zml.kernel.triton.Kernel(Config, spec)` — the declarative form + +`zml.kernel.triton.Kernel(Config, spec)` is a comptime function returning a +kernel type. `Config` is the per-call config type (used both for the +generated `Inputs`/`Outputs`/`CallOpts` and for the typed `run` signature); +`spec` is a `Spec(Config)` literal: + +| Field | Type | Meaning | +|-------------|---------------------------------------------------|---------------------------------------------------------------------| +| `.name` | `[:0]const u8` | Kernel name (used as the `tt.func` symbol). | +| `.inputs` | `[]const [:0]const u8` | Field names for the call-site `Inputs` struct (fields are `Tensor`). Order must match the `tt.func` arg order for input pointers. | +| `.outputs` | `[]const [:0]const u8` | Field names for the `Outputs` (struct of `Shape`) and `Results` (struct of `Tensor`). Order must match the `tt.func` arg order for output pointers — XLA's lowering passes inputs first, then outputs. | +| `.run` | `*const fn (*Builder, Config) FinishError!void` | Typed function pointer to the kernel body. Wrong signature → compile error here, not at first call. | + +Wrong arity, missing field, typo'd name on the call site → compile error +at the call site (because `Inputs` / `Outputs` are real struct types, not +tuples). + +`run` is responsible for declaring the kernel's args (via +`b.declareArgs(...)`) and emitting the body. It emits ops via `b.*` helpers +and never returns a string; `Kernel.emit(...)` does the +terminate-verify-serialize dance after `run` returns. + +--- + +## Calling kernels from a ZML model — `K.Kernel.call(...)` + +Inside a model's `forward(...)`, drop in a Triton kernel alongside regular +ZML ops with `K.Kernel.call(...)`: + +```zig +pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { + const x_pre = x.mul(2.0); // stablehlo + + const out = MyKernel.Kernel.call( + .{ .x_ptr = x_pre, .y_ptr = y, .n = n_tensor }, // Inputs (by name) + .{ .out = x_pre.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, // runtime config + .grid = .{ ceilDiv(x.dim(0), 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, + ).out; // .out: Tensor + + return out.relu(); // stablehlo +} +``` + +`call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. +It must be called from inside a `Compilation` (i.e., during +`zml.Compiler.compile(...)`) — `emit` builds the TTIR string in its own +throwaway context; `zml.ops.triton(...)` drops the `stablehlo.custom_call` +into the model's context. For offline tools, `Kernel.emit(allocator, cfg)` +gives you just the TTIR string. + +Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results`. + +| Positional arg | Type | Meaning | +|----------------|------------------|-------------------------------------------------------------------------| +| `inputs` | `Kernel.Inputs` | Struct of `Tensor`, one field per name in `spec.inputs`. | +| `outputs` | `Kernel.Outputs` | Struct of `Shape`, one field per name in `spec.outputs`. | + +`Kernel.CallOpts` (typed struct, all fields named): + +| Field | Type | Meaning | +|---------------------------|-----------------------|------------------------------------------| +| `.cfg` | `Kernel.Config` | Per-call kernel config. | +| `.grid` | `[3]i32` | Triton launch grid `(grid_x, grid_y, grid_z)`. | +| `.num_warps` | `i32` | Triton launch param. | +| `.num_stages` | `i32` | Triton launch param. | +| `.output_operand_aliases` | `?zml.ops.CustomCallOutputOperandAliases(Inputs, Outputs)` (default `null`) | For in-place updates: each output field names the input it aliases — `.{ .out = .x_ptr }`. | +| `.debug` | `bool` (default false)| Emit `tt.debug = true` on the custom call. | + +Returns `Kernel.Results` — a struct of `Tensor` named like `spec.outputs`. +Read by name: `out.out`, `attn_output.segm_max`, etc. + +The `cfg` value can be a comptime literal *or* a runtime expression built +from `Tensor.dim()` / `Options` fields. The kernel TTIR is regenerated per +call (cached by the MLIR canonicalizer / CSE pass at the XLA layer). + +> **Why named `Inputs` / `Outputs`.** The previous form took +> `anytype` tuples and matched by position; misordering 27 args (typical +> in `unified_attention.zig`) was a silent corruption hazard. With +> generated structs (`Inputs = StructOf(spec.inputs, Tensor)`, +> `Outputs = StructOf(spec.outputs, Shape)`), Zig refuses unknown fields, +> missing fields, and wrong-typed values at the callsite. + +--- + +## Argument specs + +Each field passed to `b.declareArgs(...)` is an `ArgSpec.Kind` tagged-union +literal. + +The four arg kinds mirror what TTIR's `tt.func` can carry: + +| Kind literal | MLIR type | Use case | +|---------------------------------------------------------|--------------------------|---------------------------------| +| `.{ .ptr = .f32 }` | `!tt.ptr` | Pointers (default divisibility) | +| `.{ .scalar = .i32 }` | plain `i32` | Strides, sizes, flags | +| `.{ .tensor = .{ &.{64, 128}, .f32 } }` | `tensor<64x128 x f32>` | Pre-built tensor input (rare) | +| `.{ .ptr_opts = .{ .dtype = .f32, .divisibility = 16 } }` | `!tt.ptr` with override | Custom `address_space`/divisibility | +| `.{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }` | plain `i32` with `tt.divisibility = 16 : i32` | Scalar with specialization hint | + +`.ptr` gets a default `tt.divisibility = 16` hint (matching Python Triton's +default). Use `.ptr_opts` to change that (or set `divisibility = null` to +suppress). `.scalar` carries no extra hint; switch to `.scalar_opts` when +you want to declare the same `tt.divisibility` Python's JIT auto-attaches +to int args whose runtime value is a multiple of 16. + +**Runtime dtypes:** the variant *tags* (`.ptr`, `.scalar`, etc.) must be +fixed per kernel — every call to `b.declareArgs(...)` for a given kernel +uses the same struct shape. The variant *payloads* (the dtype enum, the +divisibility int) can be runtime values from `cfg`: +`.a_ptr = .{ .ptr = cfg.a_dtype }` works fine even when `cfg.a_dtype` is a +runtime field. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when you need to call + into raw `ttir.*` / `arith.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through it. + The DSL populates `kernel` automatically for every value it creates (`emit`, + `arg`, loop-region args, reduction-region args, etc.). If you construct a + `Value` by hand (`.{ .inner = raw }`), the fluent methods will panic — use + `Builder.*` static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|-----------------------------------|-----------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar or tensor). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isTensor()` | `bool` | `true` for ranked tensors. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, `MAX_RANK`)| Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / ... based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | Notes | +|----------------|--------------|--------------|------------------------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | Signed integer divide. | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | Signed integer remainder. | +| `.cdiv(rhs)` | `arith.ceildivsi` | — | Int only — for `tl.cdiv`. | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | Matches `tl.minimum` (non-NaN-prop). | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | Matches `tl.maximum` (non-NaN-prop). | +| `.bitAnd(rhs)` | `arith.andi` | — | Also handles i1 masks. | +| `.bitOr(rhs)` | `arith.ori` | — | | + +### Comparisons + +All return i1 (scalar) or `tensor<... x i1>`. Signed integer predicates by +default; `.olt` / `.oge` / ... for floats. + +| Method | Int predicate | Float predicate | +|--------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts — `.to(dtype)` + +`v.to(.f32)` chooses the right MLIR op based on the current vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|------------------|-------------------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| **i1** | **float** | **`arith.uitofp`** (i1 is unsigned bool; sign-extending `1` would land at `-1.0`) | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` (no rounding) | +| float | narrower float | fp8 involved or custom rounding → `tt.fp_to_fp`; otherwise `arith.truncf` | +| same type | same type | no-op (returns `self`) | + +Triton's verifier rejects `tt.fp_to_fp` with a rounding attribute when the +target is not strictly narrower than the source (e.g. `bf16 → f32`, or +same-width `bf16 ↔ f16`). The DSL routes strict upcasts through +`arith.extf` to match Python Triton's own lowering; only genuine +downcasts carry the `rtne` rounding attribute. + +For narrowing FP→FP, the dispatch follows +`triton/python/triton/language/semantic.py:cast()`: standard FP narrowing +between fp64 / fp32 / fp16 / bf16 lowers to `arith.truncf` (rank-independent +— scalar and tensor both), while `tt.fp_to_fp` is reserved for cases where +fp8 is on either side or the caller requested a non-default +`fp_downcast_rounding`. + +### Constants + +There is no constant cache and no hoisting: every lifted literal emits its +own `arith.constant` where it is used (see Pitfall 5). `zeros(shape, dtype)`, +`ones(shape, dtype)`, `full(shape, value, dtype)` and `splat` of a literal +emit a scalar `arith.constant` and a `tt.splat`; `splat(Value, shape)` emits +the `tt.splat` alone. + +### Shape / pointer helpers + +| Method | Effect | +|------------------------------|-------------------------------------------------------------| +| `v.expandDims(axis)` / `b.expandDims(v, axis)` | `tt.expand_dims` — insert a size-1 axis at `axis`. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.broadcastTo(v, shape)` | `tt.broadcast` — explicit broadcast (size-1 dims → target). Rarely needed; auto-broadcast handles it. | +| `.splatTo(shape)` | `tt.splat` this scalar to the given shape. | +| `.addPtr(offset)` | `tt.addptr`; `offset` can be a Value or a comptime int. | + +> **Auto-broadcast is everywhere.** Mirrors Triton's +> `semantic.broadcast_impl_value` (`triton/python/triton/language/semantic.py:724`): +> rank-unification (prepend size-1 axes) + size-1 → size-N expansion. +> Applied transparently at: +> - every fluent binop (`.add`, `.mul`, `.lt`, `.bitAnd`, …) +> - `.addPtr(offset)` (either side may be scalar or rank-mismatched) +> - `b.where(cond, x, y)` / `b.select(...)` +> - `mask` and `other` of `b.loadOpts` / `b.storeOpts` +> +> So `v[:, None] * stride + w[None, :] * other_stride` is just: +> +> ```zig +> v.expandDims(1).mul(stride).add(w.expandDims(0).mul(other_stride)) +> ``` +> +> No `broadcastTo`, no full-shape `tt.splat` — the broadcast materializes +> implicitly at the `addi`. This matches Python's emit pattern op-for-op +> (Python's frontend never emits an eager `tt.broadcast` followed by a +> `arith.muli` either; it emits `expand_dims` + `splat + muli`). +> +> Reach for `b.broadcastTo` only when the consumer is a low-level op that +> bypasses the DSL's auto-broadcast — direct `arith.*` / raw `ttir.*` +> escape-hatch paths. + +--- + +## Polymorphic scalars + +Every fluent method's RHS (and every `b.splat` / `b.lift` input) is `anytype`. +The lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32, and so on. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that preserves + the source Zig width. i8/i16/i32 (signed or unsigned) → i32 const; i64/u64 → + i64 const; f16 → f16; f32 → f32; f64 → f64. Unsigned ints are bit-cast + (bit pattern preserved), so `@as(u32, 0xFFFF_FFFF)` becomes an i32 with the + same all-ones bit pattern. + +This is why the common Python idiom `offs < 64` works in Zig as +`offs.lt(64)` without any wrapper — as long as the comptime scalar can be +represented in `offs`'s element type. + +If you need to force a specific width for a runtime variable, cast at the call +site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When is `@as` actually needed? (Rule of thumb) + +Auto-lift covers most cases. `@as(...)` around a binop rhs is **redundant** +in all of these — drop it: + +| Pattern | Auto-lifts to… | +|------------------------------------------|---------------------------------------| +| `v_i64.eq(0)` / `.div(16)` / `.add(-1)` | comptime_int → match lhs elem type | +| `v_f32.mul(1.0 / config.fp8_max)` | already-f32 expression → f32 const | +| `v_i64.lt(NUM_QUERY_HEADS)` (Zig i64) | runtime i64 → i64 const | +| `v_i32.add(pad_term)` (Zig i32) | runtime i32 → i32 const | +| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / the first Value bound | +| `b.constant(.i32, 0)` | constant takes anytype — wrap nothing | + +`@as(...)` **is** required exactly when you're narrowing a runtime Zig int +to a smaller width that `lift` won't pick on its own. The idiom is +`@as(i32, @intCast(X_i64))` — you need the `@as` because `@intCast` has no +target type to infer, and without the narrowing `lift` would emit an i64 +constant, which widens the i32 lhs and the result to i64: + +```zig +// BLOCK_Q is a Zig i64 (from config). lhs is i32. Must narrow. +q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx) +``` + +The alternative — `q_start_raw.to(.i64).div(BLOCK_Q).to(.i32)` — mirrors +Python's `.to(tl.int64)` style but costs two extra Value-level casts; prefer +the narrowing `@as` or a pre-declared `const FOO_I32: i32 = @intCast(FOO);` +when the same width cast recurs. + +### When do you need `b.lift` / `b.constant`? + +Almost never. The DSL auto-lifts rhs for every fluent binop, `b.splat`, +`b.full`, `b.openFor` bounds, `.addPtr` offsets, etc. Reach for `b.lift` / +`b.constant` **only** when the DSL surface demands an existing `Value` and +there's no binop sibling to match against — typically: + +- Loop seeds for `openWhile(.{inits}, ...)` where you start with a raw + comptime literal: `const left_init = b.constant(.i32, 0);`. +- Mutable `var x: Value = b.constant(.i32, 0);` that a later branch may + reassign — the var needs an initial Value even if the comptime path would + work on its own. + +If the constant is about to flow through any `.add` / `.mul` / `.splatTo` +etc., skip the lift and let auto-lift do it. And if a stride is hardcoded +to 1, **delete it**: `x.mul(1)` is a no-op, the DSL isn't Python `constexpr` +where such names are required. + +### Scalar ↔ tensor auto-broadcast + +Fluent arithmetic and comparison ops (`.add`, `.sub`, `.mul`, `.div`, `.rem`, +`.cdiv`, `.bitAnd`, `.bitOr`, `.minimum`, `.maximum`, `.lt`, `.le`, `.gt`, +`.ge`, `.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a +scalar, the scalar is splatted to match. Either direction works: + +```zig +const offs = iv.add(cols); // scalar iv + tensor cols → tensor +const offs = cols.add(iv); // tensor cols + scalar iv → tensor +``` + +This mirrors Triton/NumPy's `i + cols`. **Size-1 and rank-unification +broadcasting also work** (mirroring Python's +`semantic.broadcast_impl_value`): a `(M, 1)` and a `(1, N)` operand +broadcast to `(M, N)` automatically, and a `(M,)` lined up against a +`(M, N)` is first `expand_dims`'d to `(1, M)` and then broadcast. So +`offs_token[:, None] + offs_k[None, :]` is just: + +```zig +const offs = b.expandDims(offs_token, 1).add(b.expandDims(offs_k, 0)); +``` + +with no `b.broadcastTo` calls needed. `b.where` / `b.select` and the +`mask` / `other` operands of `b.loadOpts` / `b.storeOpts` also auto-broadcast, +so the only time `b.broadcastTo` is needed is when the consumer bypasses +the DSL (raw `arith.*` / `ttir.*` escape-hatch paths). + +`.addPtr` auto-splats in **both directions**, matching Triton's +`ptr + offset` semantics: a scalar pointer is splatted to match a tensor +offset (`x_ptr.addPtr(tensor_offs)`), and a scalar offset is splatted to +match a tensor pointer (`a_ptrs.addPtr(BLOCK_K)`). Pre-splatting via +`b.splat` is no longer required for the loop-stride case. + +### When is `scalar.splatTo(shape)` redundant? + +Because fluent binops (`.add` / `.mul` / …) already auto-splat scalars +against tensor siblings, and `.addPtr` auto-splats either side (scalar +pointer or scalar offset) against a tensor partner, a pre-emptive +`scalar.splatTo(shape)` is usually noise: + +```zig +// redundant +pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_block.splatTo(shape)) +// equivalent — scalars auto-splat against the tensor sibling +pid_n.mul(N).add(b.arange(0, N, .i64)).rem(n_block) +``` + +Keep the explicit `splatTo` only when the scalar feeds into a raw +`arith.*` / `ttir.*` escape-hatch op that doesn't go through the DSL's +auto-broadcast. + +--- + +## Comptime vs runtime widths + +TTIR has no implicit int-width conversion — `arith.muli(i32, i64)` is a verifier +error. The DSL mirrors Python and **auto-promotes** integer binops when the +two runtime operands have different widths: it `extsi`s the narrower side +to the wider, matching `semantic.integer_promote_impl`. So +`v_i32.mul(v_i64)` works without a manual `.to(.i64)` — it emits exactly +the same `extsi` + `muli` pair Python would. + +For the rare case where you want explicit control, you can still cast +yourself; the auto-promotion is a no-op when widths already agree. + +Recurring example from `zml/kernels/moe/triton/moe.zig`: + +```zig +// pid (i32) vs block (i64 because it came from a usize config) +const block_i32: i32 = @intCast(block); +const token_start_init = pid.mul(block_i32); +// alternative: pid.to(.i64).mul(block) +``` + +For pointer math it is often cleaner to promote everything to i64 up front: + +```zig +const pid = b.programId(.x).to(.i64); +const offs = b.arange(0, BLOCK, .i64); +``` + +--- + +## Shape helpers + +| Helper | Effect | +|----------------------------------------|--------------------------------------------------| +| `b.arange(start, end, dtype)` | `tl.arange(start, end).to(dtype)`. Pass `.i32` for no-op cast. | +| `b.zeros(shape, dtype)` | `tl.zeros(shape, dtype=dtype)`. | +| `b.ones(shape, dtype)` | Ones tensor. | +| `b.splat(value, shape)` | `tl.splat`; accepts Value or comptime scalar. | +| `b.expandDims(v, axis)` / `v.expandDims(axis)` | `tt.expand_dims` — insert a size-1 axis. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.mask2d(cond_m, cond_n, _, _)` | `cond_m[:, None] & cond_n[None, :]` — emits the `(m,1) & (1,n)` form; auto-broadcast at the next consumer. | +| `b.lift(value)` | Wrap a Zig scalar as a DSL Value; dtype inferred from source. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as a DSL Value of `like`'s element type. | + +--- + +## Loads and stores + +Load helpers infer everything from the pointer's type — scalar `!tt.ptr` +loads a scalar `T`, `tensor<... x !tt.ptr>` loads a `tensor<... x T>`. +No `result_type` / `shape` / `dtype` arguments required. + +| Helper | Python equivalent | +|-------------------------------------------------|-------------------------------------------| +| `b.load(ptr)` | `tl.load(ptr)` — scalar or tensor. | +| `b.loadOpts(ptr, .{ .mask, .other, ... })` | `tl.load(ptrs, mask=..., other=..., cache_modifier=..., eviction_policy=..., volatile=...)`. | +| `b.store(ptr, value)` | `tl.store(ptr, value)`. | +| `b.storeOpts(ptr, value, .{ .mask, ... })` | `tl.store(ptr, value, mask=..., cache_modifier=..., eviction_policy=...)`. | + +`LoadOpts` / `StoreOpts` fields (`.mask`, `.other`, `.cache_modifier`, +`.eviction_policy`, `.@"volatile"`) take DSL-level `Value`s / enum literals — +no raw MLIR types needed. + +> **Match Python's emit form for masked loads.** `tl.load(ptr, mask=m, +> other=0.0)` lowers to a 3-operand `tt.load` op. To match it byte-for-byte +> in the Zig DSL, pass an explicit `.other`: +> +> ```zig +> const zero = b.zeros(&.{BLOCK}, dtype); +> const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); +> ``` +> +> Calling `loadOpts` with `.mask` only (no `.other`) emits a 2-operand +> `tt.load` — semantically equivalent (out-of-bounds lanes are undefined, +> not zero), but the TTIR/TTGIR diff vs Python will not be empty. Triton's +> `tl.load(ptr, mask=...)` without `other=` lowers to the 2-operand form, +> so you only need the `.other` operand when the Python source explicitly +> passes `other=`. + +### Result-type inference + +Most helpers that produce a new Value infer their result type from the inputs, +so user code never constructs `*const mlir.Type` manually: + +| Helper | How the result type is derived | +|-----------------------------------------------------|---------------------------------------------| +| `b.load(ptr)` / `b.loadOpts(ptr, opts)` | From `ptr.type_()` (scalar or tensor of ptrs). | +| `b.dot(a, b, acc)` / `b.dotOpts(a, b, acc, opts)` | `acc.type_()`. | +| `b.dotScaled(...)` / `b.dotScaledOpts(...)` | `acc.type_()`. | +| `b.reshape(src, new_shape)` / `b.reshapeOpts(...)` | `new_shape` + `src` element type. | +| `b.gather(src, indices, axis)` | `indices` shape + `src` element type. | +| `b.histogram(src, num_bins)` / `b.histogramOpts(...)` | `tensor`. | +| `b.fpToFp(src, out_dtype)` / `b.fpToFpOpts(...)` | `src` shape + `out_dtype`. | +| `b.convertf(src, out_dtype)` / `b.convertfOpts(...)`| `src` shape + `out_dtype`. | +| `b.scalingTruncf(src, scale, out_dtype)` / …Opts | `src` shape + `out_dtype`. | +| `b.descriptorLoad(desc, indices, shape, dtype)` | `shape` + `dtype`. | + +Rule of thumb: you only supply things the op needs beyond what inputs carry — +a `new_shape`, a `num_bins`, an `out_dtype`, a TMA tile descriptor. The DSL +never asks you to restate information it can read off a `Value`. + +`b.scalarTy(dtype)` / `b.tensorTy(shape, dtype)` are still available but only +needed for tensor result types in `openIfElse` / `openWhile` (scalar ones are +plain dtypes) and by the raw `externElementwise` / `inlineAsmElementwise` +escape hatches. + +### `fn X / fn XOpts` pattern + +Every Builder method whose Python Triton counterpart has named keyword-only +arguments comes in two flavours: + +- `b.X(non_named_args...)` — defaults applied; matches `tl.X(args...)` in + Python, which in Python would use the zero-arg defaults. +- `b.XOpts(non_named_args..., struct { named... })` — full named-param set + as a typed Zig struct literal with field defaults matching Triton. + +Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, +`dot` / `dotOpts`, `reshape` / `reshapeOpts`, +`atomicRmw` / `atomicRmwOpts`, `atomicCas` / `atomicCasOpts`, +`fpToFp` / `fpToFpOpts`, `clampf` / `clampfOpts`, +`cast` / `castOpts`, `cat` / `catOpts`, +`convertf` / `convertfOpts`, `scalingTruncf` / `scalingTruncfOpts`, +`dotScaled` / `dotScaledOpts`, +`inlineAsmElementwise` / `inlineAsmElementwiseOpts`, +`externElementwise` / `externElementwiseOpts`, +`descriptorLoad` / `descriptorLoadOpts`, `histogram` / `histogramOpts`, +`devicePrint` / `devicePrintOpts`, +`deviceAssert` / `deviceAssertOpts`, +`maximum` / `maximumOpts`, `minimum` / `minimumOpts`, +`sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, +`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`, +`openFor` / `openForOpts` (`tl.range(..., num_stages=...)`). + +DSL-only pairs (no Python kwarg counterpart — `.inline_return` is about IR +block layout, see the `returnIf` / `returnIfOpts` section under Control flow): +`returnIf` / `returnIfOpts`, `openReturnIf` / `openReturnIfOpts`. + +Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, +`trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), +`split(src)`, `join(lhs, rhs)`, `item(src)`, `broadcast(a, b)` (symmetric), +`broadcastTo(value, shape)`, `where(cond, x, y)`, `zeros(shape, dtype)`, +`full(shape, value, dtype)`. + +--- + +## Reductions and scans + +```zig +const absmax = b.max(abs_y); // tl.max(abs_y) — axis=None (all dims) +const total = b.sum(padded_counts); // tl.sum(padded_counts) +const cumul = b.cumsum(x); // tl.cumsum(x, axis=0) + +// Keyword arguments → the *Opts variant with a typed struct literal. +const axis0_max = b.maxOpts(abs_y, .{ .axis = 0 }); // tl.max(x, axis=0) +const absmax_kd = b.maxOpts(abs_y, .{ .axis = 0, .keep_dims = true }); // tl.max(x, 0, keep_dims=True) +const rev_cum = b.cumsumOpts(x, .{ .reverse = true }); // tl.cumsum(x, 0, reverse=True) +``` + +Under the hood these build the required `tt.reduce` / `tt.scan` region with a +matching `arith.addf` / `arith.addi` / `maxnumf` / `maxsi` combine function — +no more hand-rolled combiner closures for the common cases. + +> **Gotcha — `maxnumf` vs `maximumf` (and `minnumf` vs `minimumf`):** +> Triton's `tl.max`, `tl.min`, `tl.maximum`, `tl.minimum` all default to +> the **non-NaN-propagating** form (`arith.maxnumf` / `arith.minnumf`). +> The DSL fluent forms match Python: +> +> | Python | DSL | MLIR op | +> |-------------------------------------|--------------------------------------|-------------------| +> | `tl.max(x)` | `b.max(x)` / `x.max()` | `arith.maxnumf` | +> | `tl.min(x)` | `b.min(x)` / `x.min()` | `arith.minnumf` | +> | `tl.maximum(a, b)` | `a.maximum(b)` / `b.maximum(a, b)` | `arith.maxnumf` | +> | `tl.minimum(a, b)` | `a.minimum(b)` / `b.minimum(a, b)` | `arith.minnumf` | +> | `tl.maximum(a, b, propagate_nan=NAN_ALL)` | `a.maximumOpts(b, .{ .propagate_nan = .all })` | `arith.maximumf` | +> +> The lower-level builders **`b.maximumf(a, b)` / `b.minimumf(a, b)` skip +> Python's defaults** and emit IEEE-754 NaN-propagating ops directly. Reach +> for them only when you genuinely want NaN propagation; otherwise stick +> with `b.maximum` / `b.minimum` (or the fluent `Value.maximum` / +> `Value.minimum`). Mismatched ops here cause subtle TTIR/LLIR diffs vs +> Python and are a frequent porting bug. + +When `.keep_dims = true`, the DSL inserts a trailing +`tt.expand_dims` / `splat` so the result shape matches the input rank with a +size-1 reduced axis (matching Python Triton). For `axis = null` (the default), +the reduction goes through a `tt.reshape allow_reorder` first — even on rank-1 +input — so the layout-assignment pass picks the same reduction-friendly +layout Python's frontend gets. + +For custom combiners, use the low-level `b.reduce(ctx, .{ .src, .axis, .elem, +.result, .combine })` and `b.scan(ctx, .{ .src, .axis, .elem, .result, +.combine, .reverse })` — they take a `fn(*Builder, Value, Value, CtxT) Value` +closure for `combine` and give you full control. + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope +value. You emit body ops into a bare `{}` block, terminate with `yield` / +`yieldThen` / `yieldAfter` (or `close()` for a scope without values), and read +results from `scope.results` afterward. +No `ctx` struct, no function literal, no `kk`/`k` split — lexical scope +carries captures because everything runs in the same function. + +### `openFor` — scf.for with iter_args + +```zig +var loop = b.openFor(0, N, BLOCK, .{acc0}); +{ + const iv = loop.iv; + const acc = loop.carried[0]; + const offs = iv.add(cols); + const tile = b.loadOpts(x_ptr.addPtr(offs), .{ .mask = offs.lt(N) }); + const partial = tile.sum(); + loop.yield(.{ acc.add(partial) }); +} +const total = loop.results[0]; +``` + +- `lower`/`upper`/`step` are `anytype`: pass Values, comptime ints, or runtime + Zig ints. A bound that isn't a Value takes the type of the first one that is; + with none, `lower` is lifted (`i32` when it fits) and the others follow it. + Pass a typed constant (`b.constant(.i64, 0)`) to pick another width. +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an array) + all have the same length. +- Empty loops: `b.openFor(0, N, S, .{})` + `loop.close()` (no iter_args). +- `b.openForOpts(lower, upper, step, inits, .{ .num_stages = 3 })` sets the + loop's options (`tt.num_stages`, `.unsigned_cmp`). + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = b.openIf(cond); +{ + // then body + i.close(); // closes scope, builds scf.if with empty else +} +``` + +Use for conditional side-effects (no results). There is no else branch — the +else block is auto-built empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = b.openIfElse(cond, .{.f32}); // result types +{ + // then body + i.yieldThen(.{ x }); +} +{ + // else body + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `b.tensorTy(shape, dtype)`. Pass + `.{}` for an if/else with no results but distinct then/else bodies, then + end them with `elseBranch()` and `close()`. +- `yieldThen` swaps to the else block; `yieldElse` builds the `scf.if` op. +- Both branches must yield arity-matching tuples. + +### `openWhile` — scf.while + +```zig +var w = b.openWhile(.{ i0 }, .{.i32}); +{ + // before region (condition) + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); // scf.condition: cond + values forwarded to after +} +{ + // after region (body) + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); // values forwarded back to before +} +const r = w.results[0]; +``` + +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### `returnIf` / `returnIfOpts` — early `tt.return` via cf.cond_br + +```zig +// Python: if pid_m * BLOCK >= num_tokens_post_padded: return +const out_of_range = pid_m.mul(block).ge(num_tokens); +b.returnIf(out_of_range); +// ... ops here live in a fresh fall-through block ... +``` + +Lowers to: + +```mlir +cf.cond_br %out_of_range, ^ret, ^cont +^ret: + tt.return +^cont: + // subsequent ops +``` + +- Kernels return nothing, so neither form takes return values. +- Works **only at the top-level `tt.func` body** — don't call inside an + `scf.if`/`scf.for`/`scf.while` region (those must stay structured; cf + branches can't escape their parent scf region). +- After `returnIf`, the kernel's current block is swapped to a fresh + fall-through block — any subsequent `b.*` calls emit into that block, and + `b.finish()` appends the final `tt.return` there. +- Use this to match Python Triton's `if cond: return` lowering exactly. + Python emits `cf.cond_br` for early-return idioms while using `scf.for` / + `scf.if` for structured loops and value-carrying branches; our DSL does + the same. + +**`returnIfOpts(cond, .{ .inline_return = true })` — match Python's +per-`return` block layout.** By default `returnIf` routes every return path +(every early `returnIf` / `openReturnIf` plus the natural fall-through from +`b.finish`) through one **shared exit block** holding the single `tt.return`. +Python's frontend does the opposite: each `if cond: return` (and the natural +exit) gets its **own** `tt.return`-only block, and LLVM's SimplifyCFG later +merges them into a single `common.ret`. The two are semantically identical +but differ in the function-exit block's label and position — which renumbers +every `$L__BB` label downstream and blows up the PTX diff. When the Python source contains *any* literal `return` statement (it then +has ≥2 `ret` blocks in LLVM — the explicit one(s) plus the implicit +function-end one — which SimplifyCFG merges into `common.ret`), pass +`.{ .inline_return = true }` on *every* return site, including each +`openReturnIfOpts` (see below). Each one +then emits `tt.return` inline and SimplifyCFG reproduces Python's +`common.ret`. The default shared exit block emits exactly one `tt.return`, +so the merge never fires and the function-exit block lands somewhere else. + +### `openReturnIf` — early `tt.return` with side-effects in the taken branch + +```zig +// Python: if off_experts == -1: write_zeros_to_output(...); return +var scope = b.openReturnIf(is_dead); // or openReturnIfOpts, see `returnIfOpts` above +{ + writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret + scope.close(); // closes ^ret with tt.return (or cf.br ^exit) +} +// ... ops here emit into ^cont (fall-through) ... +``` + +Lowers to the same `cf.cond_br ^ret, ^cont` shape as `returnIf`, but the +taken branch can carry arbitrary side-effects between the `cf.cond_br` and +the `tt.return`. Use this instead of the +`openIf(cond){body}close; returnIf(cond)` pair — that pattern +duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same +boolean) and drifts from Triton's reference IR. + +`openReturnIfOpts(cond, .{ .inline_return = true })` makes `close` end +`^ret` with `tt.return` directly instead of `cf.br ^exit`. Use it on every +`returnIf` / `openReturnIf` of a kernel when matching a Python source that has +literal `return` statements. + +Same restrictions as `returnIf`: top-level `tt.func` body only. + +### When to use `cf` (returnIf / openReturnIf) vs `scf` (openIf / openIfElse / openFor) + +| Python source | Emits | DSL builder | +|----------------------------------------|-------|---------------------------------| +| `if cond: return` | `cf` | `b.returnIf(cond)` | +| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.close()}` | +| `if cond: ` | `scf` | `b.openIf(cond) {...; close()}` | +| `if cond: x=A else: x=B` (value-carrying) | `scf` | `b.openIfElse(cond, .{ty}) {...; yieldThen(.{a})} {...; yieldElse(.{b})}` | +| `for i in range(...)` | `scf` | `b.openFor(lo, hi, step, .{inits})` | +| `while cond:` | `scf` | `b.openWhile(.{inits}, .{after_types})` | + +Rule of thumb: **`scf` for structured, value-carrying control. `cf` for +early-exits.** Both lower to identical PTX after the +`convert-scf-to-cf` pass — the difference only matters at the TTIR layer +(pattern-matching passes that run before lowering). + +### `reduce` / `scan` — custom combiners (still callback-based) + +```zig +b.reduce(ctx, .{ .src, .axis, .elem, .result, .combine = fn (*Builder, Value, Value, Ctx) Value }); +b.scan (ctx, .{ .src, .axis, .reverse = false, .elem, .result, .combine }); +``` + +These stay on the callback pattern because the combine fn is a tiny pure +function (`(a, b) -> a ⊕ b`) where a `ctx` struct is a cleaner fit than a +scoped builder. Reach for `b.sum` / `b.max` / `b.cumsum` / etc. first — the +built-in reductions cover the common cases. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack (same for `openIfElse` / + `openWhile`). +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. +- After any `yield*`, the scope's `.results` field holds the scf op's results + as a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `ttir.*` / + `arith.*` / `math.*` / `scf.*` constructor directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its `kernel` + field populated (you'll need that for any subsequent fluent calls). + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Lower layer — `Builder.open` + +The declarative `zml.kernel.triton.Kernel(Config, spec)` form covers most +cases. For dynamic-arity arg lists, or escape-hatch scenarios where you want +to drive the IR construction yourself, use the `Builder` directly: + +```zig +const Builder = tri.Builder; + +pub fn fusedMoeKernel( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + config: GenerationConfig, +) ![:0]const u8 { + var b = try Builder.open(allocator, ctx, "fused_moe_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = config.a_dtype }, + .b_ptr = .{ .ptr = config.b_dtype }, + // ... + }); + + // ... emit body via b.* helpers; same DSL as inside `Kernel(...)` ... + + return b.finish(); +} +``` + +- `Builder.open(allocator, ctx, name)` creates an empty builder, and + `b.declareArgs(spec)` creates the `tt.func` and returns its arguments as + the same `NamedArgs(Spec)` struct as inside `Kernel(...)` — exactly what + `zml.kernel.triton.Kernel(...)` does internally. + `b.declareArgsOpts(spec, .{ .@"noinline" = true })` sets the function's + options. +- `b.finish()` terminates with `tt.return`, verifies, and serializes to a + `[:0]const u8` owned by `allocator`. `b.deinit()` frees the module. + +For dynamic-arg-count construction, `Builder.init(allocator, ctx, name, +args)` takes a runtime `[]const ArgSpec` instead of a comptime struct +literal; you grab block args with `b.arg(i)`. + +Pass these constructors a `zml.kernel.triton.newContext()`, not a +`Compilation`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR +string. `Kernel.emit` does this for you, which is why it takes no context. + +Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. + +--- + +## Python Triton ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-----------------------------------------------|--------------------------------------------------------------------| +| `tl.program_id(0)` | `b.programId(.x)` | +| `tl.num_programs(0)` | `b.numPrograms(.x)` | +| `tl.arange(0, N)` | `b.arange(0, N, .i32)` | +| `x.to(tl.int64)` | `x.to(.i64)` | +| `a + b`, `a * b`, `a // b` | `a.add(b)`, `a.mul(b)`, `a.div(b)` | +| `a < b` | `a.lt(b)` | +| `mask & other` | `mask.bitAnd(other)` | +| `tl.where(c, a, b)` | `b.where(c, a, b)` (alias of `b.select`) | +| `vec[:, None]` | `vec.expandDims(1)` | +| `vec[None, :]` | `vec.expandDims(0)` | +| `m[:, None] & (c[None, :] < N)` | `b.mask2d(m, c.lt(N), m_sz, n_sz)` | +| `ptr + offs` | `ptr.addPtr(offs)` (auto-splats scalar↔tensor either way) | +| `tl.load(ptr)` | `b.load(ptrs)` | +| `tl.load(ptr, mask=m)` | `b.loadOpts(ptrs, .{ .mask = m })` | +| `tl.load(ptr, mask=m, other=o)` | `b.loadOpts(ptrs, .{ .mask = m, .other = o })` | +| `tl.store(ptr, v)` | `b.store(ptr, v)` | +| `tl.store(ptr, v, mask=mask)` | `b.storeOpts(ptr, v, .{ .mask = mask })` | +| `tl.zeros((M, N), dtype=tl.float32)` | `b.zeros(&.{ M, N }, .f32)` | +| `tl.full((M, N), 3, tl.int32)` | `b.full(&.{ M, N }, 3, .i32)` | +| `tl.sum(x)` | `b.sum(x)` — axis=None default | +| `tl.sum(x, axis=0)` | `b.sumOpts(x, .{ .axis = 0 })` | +| `tl.sum(x, axis=0, keep_dims=True)` | `b.sumOpts(x, .{ .axis = 0, .keep_dims = true })` | +| `tl.max(x)` / `tl.min(x)` | `b.max(x)` / `b.min(x)` | +| `tl.max(x, axis=0)` | `b.maxOpts(x, .{ .axis = 0 })` | +| `tl.maximum(a, b)` / `tl.minimum(a, b)` | `a.maximum(b)` / `a.minimum(b)` or `b.maximum(a, b)` | +| `tl.cumsum(x)` / `tl.cumprod(x)` | `b.cumsum(x)` / `b.cumprod(x)` | +| `tl.cumsum(x, axis=0, reverse=True)` | `b.cumsumOpts(x, .{ .reverse = true })` | +| `tl.dot(a, b, acc=acc)` | `b.dot(a, b, acc)` — defaults to `input_precision = tf32` (NVIDIA) | +| `tl.dot(a, b, acc, input_precision="ieee")` | `b.dotOpts(a, b, acc, .{ .input_precision = .ieee })` | +| `tl.clamp(x, mn, mx)` | `b.clampf(x, mn, mx)` | +| `tl.clamp(x, mn, mx, propagate_nan=NAN_ALL)` | `b.clampfOpts(x, mn, mx, .{ .propagate_nan = .all })` | +| `tl.atomic_add(p, v)` | `b.atomicRmw(.add, p, v)` | +| `tl.atomic_add(p, v, mask=m, sem="relaxed")` | `b.atomicRmwOpts(.add, p, v, .{ .mask = m, .sem = .relaxed })` | +| `tl.histogram(x, num_bins)` | `b.histogram(x, num_bins)` | +| `tl.histogram(x, num_bins, mask=m)` | `b.histogramOpts(x, num_bins, .{ .mask = m })` | +| `tl.reshape(x, shape)` | `b.reshape(x, &.{ ...shape })` | +| `tl.reshape(x, shape, can_reorder=True)` | `b.reshapeOpts(x, &.{ ...shape }, .{ .can_reorder = true })` | +| `tl.cast(x, tl.bfloat16)` | `b.cast(x, .bf16)` (or fluent `x.to(.bf16)`) | +| `tl.cast(x, tl.bf16, fp_downcast_rounding="rtne")` | `b.castOpts(x, .bf16, .{ .fp_downcast_rounding = .rtne })` | +| `tl.cast(x, tl.int32, bitcast=True)` | `b.castOpts(x, .i32, .{ .bitcast = true })` | +| `src[indices]` (gather) | `b.gather(src, indices, axis)` | +| `tl.trans(x, dims)` / `tl.permute(x, dims)` | `b.trans(x, order)` / `b.permute(x, order)` | +| `tl.cat(a, b)` | `b.cat(a, b)` (dim=0 default) | +| `tl.cat(a, b, can_reorder=True)` | `b.catOpts(a, b, .{ .can_reorder = true })` | +| `tl.join(a, b)` / `tl.split(x)` | `b.join(a, b)` / `b.split(x)` | +| `tl.item(x)` | `b.item(x)` | +| `tl.abs(x)` | `b.abs(x)` (auto-dispatch) or fluent `x.abs()` | +| `tl.broadcast(a, b)` (symmetric) | `b.broadcast(a, b)` → `.{ Value, Value }` | +| `tl.broadcast_to(v, shape)` | `b.broadcastTo(v, &.{ ...shape })` | +| `tl.umulhi(a, b)` | `b.umulhi(a, b)` | +| `tl.sqrt_rn(x)` / `tl.div_rn(x, y)` | `b.sqrtRn(x)` / `b.divRn(x, y)` | +| `tl.device_print("dbg", args...)` | `b.devicePrint("dbg", args, is_signed)` | +| `tl.device_assert(cond, "msg")` | `b.deviceAssert(cond, "msg")` | +| `tl.inline_asm_elementwise(...)` | `b.inlineAsmElementwise(...)` | +| `tl.extern_elementwise(...)` | `b.externElementwise(srcs, shape, dtype, lib, path, sym)` | +| `for i in range(0, N, S): ...` | `var loop = b.openFor(0, N, S, .{inits...}); { ...; loop.yield(.{...}); }` | +| `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | +| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.close(); }` | +| `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | +| `if cond: return` (early exit) | `b.returnIf(cond);` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (`b.returnIfOpts(cond, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | +| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.close(); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (`b.openReturnIfOpts(cond, .{ .inline_return = true })` to match, alongside `returnIfOpts`) | + +--- + +## Pitfalls + +1. **Mixed integer widths widen.** `v_i32.mul(v_i64)` is an i64 multiply + with an `arith.extsi` on the i32 side, and so is `v_i32.mul(x)` for a + runtime Zig `i64` `x`. Narrow with `@as(i32, @intCast(x))` or `.to(.i32)` + when the result must stay i32. Comptime literals take the other operand's + type instead. + +2. **Tensor-of-pointers via `.addPtr`.** Auto-splats either side: scalar + `!tt.ptr` + tensor offset (`x_ptr.addPtr(tensor_offs)`) and tensor + `tensor<...x!tt.ptr>` + scalar offset (`a_ptrs.addPtr(BLOCK_K)`) + both work without manual `.splatTo`. + +3. **Scope-based control flow — no closures needed.** `openFor` / `openIf` / + `openIfElse` / `openWhile` push a block onto the kernel's insertion stack; + body ops emitted in the bare `{}` that follows go into that block. Captures + are plain lexical scope, so you can reference outer Values, sizes, etc. + directly — no `ctx` struct, no function literal. + +4. **`Value.kernel == null` panics fluent calls.** Only happens if you build a + `Value` by hand via `.{ .inner = raw }`. Route through `b.emit(...)` to keep + the back-pointer populated — or just stick with the explicit `Builder.*` + helpers for that one op. + +5. **Don't worry about duplicate constants.** Every lifted literal (from + `.add(42)`, `.lt(0)`, `b.splat(1.0, shape)`, an `openFor(0, N, 1, …)` + bound, etc.) emits a fresh `arith.constant`. This matches Python Triton's + own frontend — the MLIR canonicalizer + CSE pass collapses duplicates on the + way to PTX, so the generated code is identical whether you emit one + constant or a hundred. Optimize for readability at the builder level. + +6. **`finish()` verifies before serializing.** A verifier + failure here almost always means an operand-type mismatch or a + missing block terminator — a common one is forgetting that `scf.yield` + operand types must match the loop's result types. + +7. **Yielding from loop / if / while bodies.** Inside a scope (`openFor`, + `openIf`, `openIfElse`, `openWhile`), you *don't* return values — you call + `scope.yield(...)` (or `yieldThen` / `yieldElse` / `yieldBefore` / + `yieldAfter`) with a tuple of Values matching the scope's declared arity. + Scopes with results (`openFor`, `openIfElse`, `openWhile`) store the scf + op's results in `scope.results` as a fixed-size `[N]Value` array. `openIf` + has no results — `close()` closes and builds scf.if with an empty + else block in one step. diff --git a/dsl/triton/builder.zig b/dsl/triton/builder.zig new file mode 100644 index 00000000..5ccec273 --- /dev/null +++ b/dsl/triton/builder.zig @@ -0,0 +1,2132 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + pub fn isTensor(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + /// Ceil-div: `(x + (div - 1)) // div`. Comptime rhs folds `rhs - 1`. + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return switch (@typeInfo(@TypeOf(rhs))) { + .comptime_int, .comptime_float => k.divsi(l.add(rhs - 1), r), + else => k.divsi(k.addi(l, r.sub(1)), r), + }; + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn minimumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().minimumOpts(self, rhs, opts); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + pub fn maximumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().maximumOpts(self, rhs, opts); + } + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + pub fn toOpts(self: Value, dtype: DType, opts: CastOpts) Value { + return self.kern().castOpts(self, dtype, opts); + } + + pub fn expandDims(self: Value, axis: i32) Value { + return self.kern().expandDims(self, axis); + } + + pub fn splatTo(self: Value, shape_: []const i64) Value { + return self.kern().splat(self, shape_); + } + + pub fn sum(self: Value) Value { + return self.kern().sum(self); + } + pub fn sumOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().sumOpts(self, opts); + } + + pub fn max(self: Value) Value { + return self.kern().max(self); + } + pub fn maxOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().maxOpts(self, opts); + } + + pub fn min(self: Value) Value { + return self.kern().min(self); + } + pub fn minOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().minOpts(self, opts); + } + + pub fn cumsum(self: Value) Value { + return self.kern().cumsum(self); + } + pub fn cumsumOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumsumOpts(self, opts); + } + + pub fn cumprod(self: Value) Value { + return self.kern().cumprod(self); + } + pub fn cumprodOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumprodOpts(self, opts); + } + + pub fn gather(self: Value, indices: Value, axis: i32) Value { + return self.kern().gather(self, indices, axis); + } + + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } + + pub fn addPtr(self: Value, offset: anytype) Value { + const k = self.kern(); + const off: Value = if (@TypeOf(offset) == Value) offset else k.lift(offset); + const ptr = if (!self.isTensor() and off.isTensor()) k.splat(self, off.shape().constSlice()) else self; + const off2 = if (self.isTensor() and !off.isTensor()) k.splat(off, self.shape().constSlice()) else off; + return k.addptr(ptr, off2); + } +}; + +fn computeReducedType(src_shape: []const i64, axis: i32, elem: *const mlir.Type) *const mlir.Type { + if (src_shape.len <= 1) return elem; + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (src_shape, 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + return .rankedTensor(out.constSlice(), elem); +} + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + ptr: DType, + scalar: DType, + tensor: struct { []const i64, DType }, + ptr_opts: PtrOpts, + scalar_opts: ScalarOpts, + }; + + pub const PtrOpts = struct { + dtype: DType, + address_space: i32 = 1, + divisibility: ?u32 = 16, + }; + + pub const ScalarOpts = struct { + dtype: DType, + divisibility: ?u32 = null, + }; +}; + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const ClampOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const HistogramOpts = struct { + mask: ?Value = null, +}; + +pub const PrintOpts = struct { + hex: bool = false, +}; + +/// `axis=null` reduces all dims. +pub const ReduceOpts = struct { + axis: ?i32 = null, + keep_dims: bool = false, +}; + +pub const ScanOpts = struct { + axis: i32 = 0, + reverse: bool = false, +}; + +pub const LoadOpts = struct { + mask: ?Value = null, + other: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, + @"volatile": bool = false, +}; + +pub const StoreOpts = struct { + mask: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const ReshapeOpts = struct { + can_reorder: bool = false, + efficient_layout: bool = false, +}; + +pub const DotOpts = struct { + input_precision: ttir.InputPrecision = .tf32, + max_num_imprecise_acc: i32 = 0, +}; + +pub const FpToFpOpts = struct { + rounding: ?ttir.RoundingMode = null, +}; + +pub const AtomicRMWOpts = struct { + mask: ?Value = null, + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const AtomicCasOpts = struct { + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const DotScaledOpts = struct { + fast_math: bool = false, + lhs_k_pack: bool = true, + rhs_k_pack: bool = true, +}; + +pub const ExternElementwiseOpts = struct { + is_pure: bool = true, +}; + +pub const DescriptorLoadOpts = struct { + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const InlineAsmOpts = struct { + is_pure: bool = true, + pack: i32 = 1, +}; + +pub const CatOpts = struct { + can_reorder: bool = false, + dim: i32 = 0, +}; + +pub const CastOpts = struct { + fp_downcast_rounding: ?ttir.RoundingMode = null, + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const DeviceAssertOpts = struct { + mask: ?Value = null, +}; + +pub const ReturnIfOpts = struct { + inline_return: bool = false, +}; + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +/// `if (cond) { ...; return; }`: `close` returns at the end of the body, +/// then the kernel continues after it. +pub const ReturnIfScope = struct { + kernel: *Builder, + ret_block: *mlir.Block, + cont_block: *mlir.Block, + opts: ReturnIfOpts, + + pub fn close(self: *ReturnIfScope) void { + const k = self.kernel; + if (self.opts.inline_return) { + _ = ttir.return_(k.ctx, &.{}, k.loc()).appendTo(self.ret_block); + } else { + _ = cf.br(k.ctx, k.exitBlock(), &.{}, k.loc()).appendTo(self.ret_block); + } + k.popBlock(); + k.continueIn(self.cont_block); + } +}; + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + reverse: bool = false, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, ttir, arith, scf, math, cf, llvm }) |dialect| dialect.registerDialect(registry); +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + exit_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + + /// `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + }; + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + ) !Builder { + return initOpts(allocator, ctx, name, args, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + opts: Opts, + ) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .scalar => .{ .scalar = inner }, + .tensor => .{ .tensor = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .address_space = if (@hasField(@TypeOf(inner), "address_space")) inner.address_space else 1, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else 16, + } }, + .scalar_opts => .{ .scalar_opts = .{ + .dtype = inner.dtype, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else null, + } }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { + .ptr => |dt| ttir.pointerType(dt.toMlir(ctx), 1), + .ptr_opts => |p| ttir.pointerType(p.dtype.toMlir(ctx), p.address_space), + .scalar => |dt| dt.toMlir(ctx), + .scalar_opts => |s| s.dtype.toMlir(ctx), + .tensor => |t| .rankedTensor(t[0], t[1].toMlir(ctx)), + }; + a_loc.* = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + var any_arg_attr = false; + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .ptr => .dict(ctx, &.{ttir.divisibility(ctx, 16)}), + .ptr_opts => |p| if (p.divisibility) |v| + .dict(ctx, &.{ttir.divisibility(ctx, v)}) + else + empty_dict, + .scalar_opts => |s| if (s.divisibility) |v| + .dict(ctx, &.{ttir.divisibility(ctx, v)}) + else + empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; + } + + const func_op = ttir.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{}, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn tensorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return .rankedTensor(shape, dtype.toMlir(self.ctx)); + } + + fn loadResultType(self: *const Builder, ptr: Value) *const mlir.Type { + _ = self; + const pt = ptr.type_(); + if (pt.isA(mlir.ShapedType)) |shaped| { + const elem_ptr_ty = shaped.elementType(); + const pointee = elem_ptr_ty.isA(ttir.PointerType).?.pointee(); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..shaped.rank()) |i| shape.appendAssumeCapacity(shaped.dimension(i)); + return .rankedTensor(shape.constSlice(), pointee); + } + return pt.isA(ttir.PointerType).?.pointee(); + } + + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isTensor()) return .rankedTensor(src.shape().constSlice(), out_elem); + return out_elem; + } + + // ==================== SPMD ==================== + + pub fn programId(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_program_id(self.ctx, dim, self.loc())); + } + + pub fn numPrograms(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_num_programs(self.ctx, dim, self.loc())); + } + + pub fn assume(self: *Builder, cond: Value) void { + _ = llvm.intr_assume(self.ctx, cond.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + else => self.fpToFp(self.emitFloat(.f32, value), dtype), + }; + } + + // ==================== ranges and shape manipulation ==================== + + /// `[start, end)` as `tensor<(end-start)xi32>`. Both must fit in i32. + pub fn makeRange(self: *Builder, start: anytype, end: anytype) Value { + const s: i32 = @intCast(start); + const e: i32 = @intCast(end); + std.debug.assert(e >= s); + const len: i64 = @intCast(e - s); + const ty = mlir.Type.rankedTensor(&.{len}, .int(self.ctx, .i32)); + return self.emit(ttir.make_range(self.ctx, s, e, ty, self.loc())); + } + + pub fn arange(self: *Builder, start: anytype, end: anytype, dtype: DType) Value { + const r = self.makeRange(start, end); + return switch (dtype) { + .i32 => r, + .i64 => self.extsi(r, dtype), + .i16, .i8 => self.trunci(r, dtype), + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => self.sitofp(r, dtype), + else => @panic("Builder.arange: unsupported dtype"), + }; + } + + /// Broadcast a scalar Value or literal across `shape`. Literals lift first via `lift`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64) Value { + const v: Value = if (@TypeOf(value) == Value) value else self.lift(value); + const ty: *const mlir.Type = .rankedTensor(shape, v.type_()); + return self.emit(ttir.splat(self.ctx, v.inner, ty, self.loc())); + } + + /// Lift a Zig scalar to a DSL constant. Dtype follows the source type; + /// unsigned ints are bit-cast (signless MLIR). Use `constant` for an explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `lift` but comptime scalars match `ref_elem`. Runtime Zig scalars keep their source width. + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Lift `value` and splat to `ref.shape` when `ref` is a tensor. + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + if (ref.isTensor() and !v.isTensor()) return self.splat(v, ref.shape().constSlice()); + return v; + } + + /// Binary-op coercion: lift, int-width promote, scalar↔tensor splat, rank-align, size-1 broadcast. + pub fn broadcast(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + if (av.isTensor() and !bv.isTensor()) bv = self.splat(bv, av.shape().constSlice()); + if (!av.isTensor() and bv.isTensor()) av = self.splat(av, bv.shape().constSlice()); + + if (av.isTensor() and bv.isTensor()) { + const a_rank = av.rank(); + const b_rank = bv.rank(); + if (a_rank < b_rank) { + var n = b_rank - a_rank; + while (n > 0) : (n -= 1) av = self.expandDims(av, 0); + } else if (b_rank < a_rank) { + var n = a_rank - b_rank; + while (n > 0) : (n -= 1) bv = self.expandDims(bv, 0); + } + const a_sh = av.shape(); + const b_sh = bv.shape(); + var ret_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + var grow_a = false; + var grow_b = false; + for (0..a_sh.len) |i| { + const l = a_sh.get(i); + const r = b_sh.get(i); + if (l == r) { + ret_shape.appendAssumeCapacity(l); + } else if (l == 1) { + ret_shape.appendAssumeCapacity(r); + grow_a = true; + } else if (r == 1) { + ret_shape.appendAssumeCapacity(l); + grow_b = true; + } else { + std.debug.panic( + "Builder.broadcast: incompatible shapes {any} vs {any} at axis {d}", + .{ a_sh.constSlice(), b_sh.constSlice(), i }, + ); + } + } + if (grow_a) av = self.broadcastTo(av, ret_shape.constSlice()); + if (grow_b) bv = self.broadcastTo(bv, ret_shape.constSlice()); + } + + return .{ av, bv }; + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(self.ctx))) return dt; + } + @panic("element type not a recognized DSL DType"); + } + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); + } + + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + return self.constant(self.mlirElemToDType(elem), value); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.constant(dtype, value), shape); + } + + /// 2-D mask `(m, 1) & (1, n)` — auto-broadcasts to `(m, n)`. + pub fn mask2d(self: *Builder, cond_m: Value, cond_n: Value, _: i64, _: i64) Value { + const cm2 = self.expandDims(cond_m, 1); + const cn2 = self.expandDims(cond_n, 0); + return cm2.bitAnd(cn2); + } + + pub fn sum(self: *Builder, src: Value) Value { + return self.sumOpts(src, .{}); + } + + pub fn sumOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .sum); + } + + pub fn max(self: *Builder, src: Value) Value { + return self.maxOpts(src, .{}); + } + + pub fn maxOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .max); + } + + pub fn min(self: *Builder, src: Value) Value { + return self.minOpts(src, .{}); + } + + pub fn minOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .min); + } + + fn reduceDispatch(self: *Builder, src: Value, opts: ReduceOpts, comptime kind: enum { sum, max, min }) Value { + const src_shape = src.shape(); + const input: Value, const axis: i32 = if (opts.axis) |a| .{ src, a } else blk: { + var total: i64 = 1; + for (0..src_shape.len) |i| total *= src_shape.get(i); + break :blk .{ self.reshapeOpts(src, &.{total}, .{ .can_reorder = true }), 0 }; + }; + + const elem = input.elemType(); + const in_shape = input.shape(); + const result_ty: *const mlir.Type = computeReducedType(in_shape.constSlice(), axis, elem); + const is_float = input.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .max => if (cc.is_float) kk.maxnumf(lhs, rhs) else kk.maxsi(lhs, rhs), + .min => if (cc.is_float) kk.minnumf(lhs, rhs) else kk.minsi(lhs, rhs), + }; + } + }.c; + const out = self.reduce(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = input, + .axis = axis, + .elem = elem, + .result = result_ty, + .combine = combine, + }); + + if (!opts.keep_dims) return out; + if (opts.axis == null) { + var ones_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..src_shape.len) |_| ones_shape.appendAssumeCapacity(1); + return self.splat(out, ones_shape.constSlice()); + } + if (in_shape.len <= 1) return self.splat(out, &.{1}); + return self.expandDims(out, axis); + } + + pub fn cumsum(self: *Builder, src: Value) Value { + return self.cumsumOpts(src, .{}); + } + + pub fn cumsumOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .sum); + } + + pub fn cumprod(self: *Builder, src: Value) Value { + return self.cumprodOpts(src, .{}); + } + + pub fn cumprodOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .prod); + } + + fn scanDispatch(self: *Builder, src: Value, opts: ScanOpts, comptime kind: enum { sum, prod }) Value { + const elem = src.elemType(); + const is_float = src.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .prod => if (cc.is_float) kk.mulf(lhs, rhs) else kk.muli(lhs, rhs), + }; + } + }.c; + return self.scan(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = src, + .axis = opts.axis, + .reverse = opts.reverse, + .elem = elem, + .result = src.type_(), + .combine = combine, + }); + } + + pub fn expandDims(self: *Builder, value: Value, axis: i32) Value { + const in_shape = value.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + const ax: usize = @intCast(axis); + for (0..in_shape.len + 1) |i| { + if (i == ax) out_shape.appendAssumeCapacity(1) else out_shape.appendAssumeCapacity(in_shape.get(if (i < ax) i else i - 1)); + } + const ty = mlir.Type.rankedTensor(out_shape.constSlice(), value.elemType()); + return self.emit(ttir.expand_dims(self.ctx, value.inner, axis, ty, self.loc())); + } + + pub fn broadcastTo(self: *Builder, value: Value, result_shape: []const i64) Value { + const elem = mlir.RankedTensorType.fromShaped(value.inner.type_().isA(mlir.ShapedType).?).elementType(); + const ty = mlir.Type.rankedTensor(result_shape, elem); + return self.emit(ttir.broadcast(self.ctx, value.inner, ty, self.loc())); + } + + // ==================== pointer arithmetic ==================== + + pub fn addptr(self: *Builder, ptr: Value, offset: Value) Value { + return self.emit(ttir.addptr(self.ctx, ptr.inner, offset.inner, self.loc())); + } + + pub fn load(self: *Builder, ptr: Value) Value { + return self.loadOpts(ptr, .{}); + } + + pub fn loadOpts(self: *Builder, ptr: Value, opts: LoadOpts) Value { + var mask_v = opts.mask; + var other_v = opts.other; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + if (other_v) |o| { + _, other_v = self.broadcast(ptr, o); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + const other_inner: ?*const mlir.Value = if (other_v) |o| o.inner else null; + return self.emit(ttir.load( + self.ctx, + ptr.inner, + self.loadResultType(ptr), + opts.cache_modifier, + opts.eviction_policy, + opts.@"volatile", + .{ .mask = mask_inner, .other = other_inner }, + self.loc(), + )); + } + + pub fn store(self: *Builder, ptr: Value, value: Value) void { + self.storeOpts(ptr, value, .{}); + } + + pub fn storeOpts(self: *Builder, ptr: Value, value: Value, opts: StoreOpts) void { + var mask_v = opts.mask; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + _ = ttir.store(self.ctx, ptr.inner, value.inner, opts.cache_modifier, opts.eviction_policy, .{ .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + /// Select `x` where condition is true, `y` elsewhere. Auto-broadcasts. + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + var c = condition; + var xv, var yv = .{ x, y }; + xv, yv = self.broadcast(xv, yv); + if (c.isTensor()) { + c, xv = self.broadcast(c, xv); + xv, yv = self.broadcast(xv, yv); + } else { + c, _ = self.broadcast(c, xv); + } + // Arithmetic broadcasting can widen i1 when the selected values are + // integers. The select predicate must remain boolean. + return self.select(c.to(.i1), xv, yv); + } + + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ceildivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ceildivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn floordivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.floordivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.maxnumf(l, r), + .all => self.maximumf(l, r), + }; + } + return self.maxsi(l, r); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.minnumf(l, r), + .all => self.minimumf(l, r), + }; + } + return self.minsi(l, r); + } + + // Casts — each preserves the src shape and swaps the element type to `out_dtype`. + pub fn extsi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + // ==================== Triton-specific ops ==================== + + /// tt.bitcast — same-bitwidth cast. Shape preserved from src. + pub fn bitCast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch. Use `castOpts` for rounding / bitcast. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + return self.castOpts(src, dtype, .{}); + } + + /// Numeric cast with options. Dispatch: + /// - fp↔fp with fp8: tt.fp_to_fp, with rounding only on downcasts. + /// - other fp↔fp narrow: truncf, except custom rounding → tt.fp_to_fp. + /// - other fp↔fp wide: extf. + /// - i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. + pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitCast(src, dtype); + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e4m3fnuz or cur_dtype == .f8e5m2 or + dtype == .f8e4m3fn or dtype == .f8e4m3fnuz or dtype == .f8e5m2); + if (fp8_involved) { + return self.fpToFpOpts(src, dtype, .{ + .rounding = if (tgt_bw < cur_bw) opts.fp_downcast_rounding orelse .rtne else null, + }); + } + if (tgt_bw > cur_bw) return self.extf(src, dtype); + const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; + if (custom_rounding) { + return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); + } + return self.truncf(src, dtype); + } + if (cur_is_float and !tgt_is_float) return self.fptosi(src, dtype); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.uitofp(src, dtype); + return self.sitofp(src, dtype); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) { + if (cur_dtype == .i1) return self.extui(src, dtype); + return self.extsi(src, dtype); + } + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.arithBitcast(src, dtype); + } + /// tt.int_to_ptr — cast int (scalar or tensor) to pointer, preserving shape. + pub fn intToPtr(self: *Builder, src: Value, pointee: DType, address_space: i32) Value { + const ptr_elem = ttir.pointerType(pointee.toMlir(self.ctx), address_space); + const result_type: *const mlir.Type = if (src.isTensor()) + .rankedTensor(src.shape().constSlice(), ptr_elem) + else + ptr_elem; + return self.emit(ttir.int_to_ptr(self.ctx, src.inner, result_type, self.loc())); + } + /// tt.ptr_to_int — cast pointer (scalar or tensor) to integer, preserving shape. + pub fn ptrToInt(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.ptr_to_int(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + /// tt.fp_to_fp with default rtne rounding. Use `fpToFpOpts` for other rounding modes. + pub fn fpToFp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.fpToFpOpts(src, out_dtype, .{ .rounding = .rtne }); + } + /// tt.fp_to_fp with rounding option. + pub fn fpToFpOpts(self: *Builder, src: Value, out_dtype: DType, opts: FpToFpOpts) Value { + return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), .{ .rounding = opts.rounding }, self.loc())); + } + /// tt.clampf with propagate_nan=.none. Use `clampfOpts` to override. + pub fn clampf(self: *Builder, x: Value, lo: Value, hi: Value) Value { + return self.clampfOpts(x, lo, hi, .{}); + } + /// tt.clampf with full options. + pub fn clampfOpts(self: *Builder, x: Value, lo: Value, hi: Value, opts: ClampOpts) Value { + return self.emit(ttir.clampf(self.ctx, x.inner, lo.inner, hi.inner, opts.propagate_nan, self.loc())); + } + /// `tl.sqrt_rn(x)` — precise square root (IEEE round-to-nearest). + pub fn sqrtRn(self: *Builder, x: Value) Value { + return self.emit(ttir.precise_sqrt(self.ctx, x.inner, self.loc())); + } + /// `tl.div_rn(x, y)` — precise division (IEEE round-to-nearest). + pub fn divRn(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.precise_divf(self.ctx, x.inner, y.inner, self.loc())); + } + /// `tl.umulhi(x, y)` — most-significant N bits of the 2N-bit unsigned product. + pub fn umulhi(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.mulhiui(self.ctx, x.inner, y.inner, self.loc())); + } + + /// tt.dot — matmul with accumulator. Result type = c_acc. Use `dotOpts` for precision. + pub fn dot(self: *Builder, a: Value, b: Value, c_acc: Value) Value { + return self.dotOpts(a, b, c_acc, .{}); + } + + /// tt.dot with full options. Result type = `c_acc.type_()`. + pub fn dotOpts(self: *Builder, a: Value, b: Value, c_acc: Value, opts: DotOpts) Value { + return self.emit(ttir.dot(self.ctx, a.inner, b.inner, c_acc.inner, c_acc.type_(), opts.input_precision, opts.max_num_imprecise_acc, self.loc())); + } + + /// tt.reshape, element type preserved. Use `reshapeOpts` for can_reorder / efficient_layout. + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + return self.reshapeOpts(src, new_shape, .{}); + } + + /// tt.reshape with full options. + pub fn reshapeOpts(self: *Builder, src: Value, new_shape: []const i64, opts: ReshapeOpts) Value { + const result_ty = mlir.Type.rankedTensor(new_shape, src.elemType()); + return self.emit(ttir.reshape(self.ctx, src.inner, result_ty, opts.can_reorder, opts.efficient_layout, self.loc())); + } + + /// tt.trans — permute dimensions. Result shape is src.shape permuted by order. + pub fn trans(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (order) |i| out_shape.appendAssumeCapacity(src_shape.get(@intCast(i))); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); + return self.emit(ttir.trans(self.ctx, src.inner, order, result_type, self.loc())); + } + + /// `tl.permute(src, *dims)` — alias for `trans`. + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + return self.trans(src, order); + } + + /// `tl.cat` along dim 0. Use `catOpts` for can_reorder / dim. + pub fn cat(self: *Builder, lhs: Value, rhs: Value) Value { + return self.catOpts(lhs, rhs, .{}); + } + + /// `tl.cat` with full options: `join`, `permute` and `reshape`, as Triton lowers it. + pub fn catOpts(self: *Builder, lhs: Value, rhs: Value, opts: CatOpts) Value { + _ = opts.can_reorder; + const ls = lhs.shape(); + const rs = rhs.shape(); + std.debug.assert(ls.len == rs.len); + for (0..ls.len) |i| std.debug.assert(ls.get(i) == rs.get(i)); + const rank: i32 = @intCast(ls.len); + const dim: usize = @intCast(if (opts.dim < 0) opts.dim + rank else opts.dim); + var order: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| order.appendAssumeCapacity(@intCast(i)); + order.insert(dim, rank) catch unreachable; + var new_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| new_shape.appendAssumeCapacity(ls.get(i)); + new_shape.slice()[dim] = ls.get(dim) + rs.get(dim); + return self.reshape(self.permute(self.join(lhs, rhs), order.constSlice()), new_shape.constSlice()); + } + + /// `tt.join` — stack two equal-shape tensors along a new minor dim of size 2. + pub fn join(self: *Builder, lhs: Value, rhs: Value) Value { + const ls = lhs.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); + out_shape.appendAssumeCapacity(2); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); + return self.emit(ttir.join(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); + } + + /// `tt.split` — split a tensor whose last dim is 2 into two tensors with + /// that last dim dropped. Returns (lhs, rhs). + pub fn split(self: *Builder, src: Value) [2]Value { + const ss = src.shape(); + std.debug.assert(ss.len >= 1 and ss.get(ss.len - 1) == 2); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ss.len - 1) |i| out_shape.appendAssumeCapacity(ss.get(i)); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); + const op = ttir.split(self.ctx, src.inner, result_type, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + }; + } + + /// `tl.item(src)` / `tt.unsplat` — 1-element tensor → scalar of the same element type. + pub fn item(self: *Builder, src: Value) Value { + return self.emit(ttir.unsplat(self.ctx, src.inner, src.elemType(), self.loc())); + } + + /// tt.gather — output shape matches indices, element type matches src. + pub fn gather(self: *Builder, src: Value, indices: Value, axis: i32) Value { + const result_ty = mlir.Type.rankedTensor(indices.shape().constSlice(), src.elemType()); + return self.emit(ttir.gather(self.ctx, src.inner, indices.inner, axis, result_ty, false, self.loc())); + } + + /// tt.histogram — output is `tensor`. Use `histogramOpts` for mask. + pub fn histogram(self: *Builder, src: Value, num_bins: i64) Value { + return self.histogramOpts(src, num_bins, .{}); + } + + /// tt.histogram with mask option. + pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { + const result_ty = mlir.Type.rankedTensor(&.{num_bins}, DType.i32.toMlir(self.ctx)); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.histogram(self.ctx, src.inner, result_ty, .{ .mask = mask_inner }, self.loc())); + } + + /// tt.assert — device-side assert (i1 or tensor<...xi1>). Use `deviceAssertOpts` for mask. + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + self.deviceAssertOpts(condition, message, .{}); + } + + /// tt.assert with mask option. + pub fn deviceAssertOpts(self: *Builder, condition: Value, message: []const u8, opts: DeviceAssertOpts) void { + // TODO: threading mask through ttir.assert_ when supported. + _ = opts.mask; + _ = ttir.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.atomic_rmw — returns old value. Use `atomicRmwOpts` for mask / sem / scope. + pub fn atomicRmw(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value) Value { + return self.atomicRmwOpts(rmw, ptr, val, .{}); + } + + /// tt.atomic_rmw with full options. + pub fn atomicRmwOpts(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value, opts: AtomicRMWOpts) Value { + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, opts.sem, opts.scope, .{ .mask = mask_inner }, self.loc())); + } + + /// tt.atomic_cas — returns old value. Use `atomicCasOpts` for sem / scope. + pub fn atomicCas(self: *Builder, ptr: Value, cmp: Value, val: Value) Value { + return self.atomicCasOpts(ptr, cmp, val, .{}); + } + + /// tt.atomic_cas with full options. + pub fn atomicCasOpts(self: *Builder, ptr: Value, cmp: Value, val: Value, opts: AtomicCasOpts) Value { + return self.emit(ttir.atomic_cas(self.ctx, ptr.inner, cmp.inner, val.inner, opts.sem, opts.scope, self.loc())); + } + + /// `tt.call` — call another tt.func in this module by symbol name. + pub fn call(self: *Builder, callee: []const u8, operands: []const Value, result_types: []const *const mlir.Type) []Value { + const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, .{}, self.loc()); + return self.emitMulti(op, result_types.len); + } + + /// tt.dot_scaled — dot with microscaling factors. Use `dotScaledOpts` for options. + pub fn dotScaled( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + ) Value { + return self.dotScaledOpts(a, b, c_acc, a_scale, b_scale, a_elem_type, b_elem_type, .{}); + } + + /// `tt.dot_scaled` with the full named-param set. Result type = `c_acc.type_()`. + pub fn dotScaledOpts( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + opts: DotScaledOpts, + ) Value { + const a_s: ?*const mlir.Value = if (a_scale) |s| s.inner else null; + const b_s: ?*const mlir.Value = if (b_scale) |s| s.inner else null; + return self.emit(ttir.dot_scaled( + self.ctx, + a.inner, + b.inner, + c_acc.inner, + a_elem_type, + b_elem_type, + c_acc.type_(), + opts.fast_math, + opts.lhs_k_pack, + opts.rhs_k_pack, + .{ .a_scale = a_s, .b_scale = b_s }, + self.loc(), + )); + } + + /// tt.extern_elementwise — call a library symbol pointwise. + /// Result is `tensor`; pass `&.{}` for scalar. + pub fn externElementwise( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + ) Value { + return self.externElementwiseOpts(srcs, result_shape, result_dtype, libname, libpath, symbol, .{}); + } + + /// `tt.extern_elementwise` with the full named-param set. + pub fn externElementwiseOpts( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + opts: ExternElementwiseOpts, + ) Value { + const result_type: *const mlir.Type = if (result_shape.len == 0) + result_dtype.toMlir(self.ctx) + else + .rankedTensor(result_shape, result_dtype.toMlir(self.ctx)); + return self.emit(ttir.extern_elementwise( + self.ctx, + self.innerSlice(srcs), + result_type, + libname, + libpath, + symbol, + opts.is_pure, + self.loc(), + )); + } + + /// tt.print — device-side print. `is_signed` has one entry per arg (1=signed, 0=unsigned). + /// Use `devicePrintOpts` for hex. + pub fn devicePrint( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + ) void { + self.devicePrintOpts(prefix, args, is_signed, .{}); + } + + /// tt.print with full options. + pub fn devicePrintOpts( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + opts: PrintOpts, + ) void { + _ = ttir.print(self.ctx, prefix, opts.hex, self.innerSlice(args), is_signed, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.make_tensor_descriptor — build a TMA descriptor. + /// shape/strides are runtime; block_shape/dtype describe the compile-time tile. + pub fn makeTensorDescriptor( + self: *Builder, + base: Value, + shape: []const Value, + strides: []const Value, + padding: ttir.PaddingOption, + block_shape: []const i64, + dtype: DType, + ) Value { + const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), .{}); + return self.emit(ttir.make_tensor_descriptor( + self.ctx, + base.inner, + self.innerSlice(shape), + self.innerSlice(strides), + padding, + result_type, + self.loc(), + )); + } + + /// tt.descriptor_load — TMA load. Use `descriptorLoadOpts` for cache / eviction. + pub fn descriptorLoad( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + ) Value { + return self.descriptorLoadOpts(desc, indices, shape, dtype, .{}); + } + + /// `tt.descriptor_load` with the full named-param set. + pub fn descriptorLoadOpts( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + opts: DescriptorLoadOpts, + ) Value { + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_load(self.ctx, desc.inner, self.innerSlice(indices), result_ty, opts.cache_modifier, opts.eviction_policy, self.loc())); + } + + /// `tt.descriptor_store` — TMA store. + pub fn descriptorStore(self: *Builder, desc: Value, src: Value, indices: []const Value) void { + _ = ttir.descriptor_store(self.ctx, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// `tt.descriptor_reduce` — TMA reducing store. + pub fn descriptorReduce( + self: *Builder, + kind: ttir.DescriptorReduceKind, + desc: Value, + src: Value, + indices: []const Value, + ) void { + _ = ttir.descriptor_reduce(self.ctx, kind, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// tt.descriptor_gather — TMA gather by x_offsets + y_offset. + pub fn descriptorGather( + self: *Builder, + desc: Value, + x_offsets: Value, + y_offset: Value, + shape: []const i64, + dtype: DType, + ) Value { + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_gather(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, result_ty, self.loc())); + } + + /// `tt.descriptor_scatter` — TMA scatter. + pub fn descriptorScatter(self: *Builder, desc: Value, x_offsets: Value, y_offset: Value, src: Value) void { + _ = ttir.descriptor_scatter(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, src.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(math.tan(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + /// `tl.abs(x)` — absolute value, auto-dispatch by element type (float→`absf`, + /// signed int→`absi`, unsigned int→no-op). + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + /// `math.clampf` — distinct from `tt.clampf` (no propagateNan attr). + pub fn mathClampf(self: *Builder, value: Value, lo: Value, hi: Value) Value { + return self.emit(math.clampf(self.ctx, value.inner, lo.inner, hi.inner, self.loc())); + } + /// `math.sincos` — returns (sin, cos). + pub fn sincos(self: *Builder, x: Value) [2]Value { + const op = math.sincos(self.ctx, x.inner, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .{ .inner = op.result(0), .kernel = self }, .{ .inner = op.result(1), .kernel = self } }; + } + /// `math.fpowi` — float base, integer exponent. + pub fn fpowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.fpowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn ipowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.ipowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn atan2(self: *Builder, y: Value, x: Value) Value { + return self.emit(math.atan2(self.ctx, y.inner, x.inner, self.loc())); + } + pub fn copysign(self: *Builder, mag: Value, sign: Value) Value { + return self.emit(math.copysign(self.ctx, mag.inner, sign.inner, self.loc())); + } + pub fn mathRound(self: *Builder, x: Value) Value { + return self.emit(math.round(self.ctx, x.inner, self.loc())); + } + pub fn mathRoundEven(self: *Builder, x: Value) Value { + return self.emit(math.roundeven(self.ctx, x.inner, self.loc())); + } + pub fn mathTrunc(self: *Builder, x: Value) Value { + return self.emit(math.trunc(self.ctx, x.inner, self.loc())); + } + + /// `arith.convertf` — same-bitwidth float cast (e.g. f16 ↔ bf16) with + /// default rounding. Shape preserved from `src`. Use `convertfOpts` for + /// `fastMath`. + pub fn convertf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.convertfOpts(src, out_dtype, .{}); + } + /// `arith.convertf` with the full named-param set. + pub fn convertfOpts(self: *Builder, src: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.convertf(self.ctx, src.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + /// `arith.scaling_truncf` — MXFP downcast with per-block scales, default + /// options. Shape preserved from `src`. Use `scalingTruncfOpts` for + /// `fastMath`. + pub fn scalingTruncf(self: *Builder, src: Value, scale: Value, out_dtype: DType) Value { + return self.scalingTruncfOpts(src, scale, out_dtype, .{}); + } + /// `arith.scaling_truncf` with the full named-param set. + pub fn scalingTruncfOpts(self: *Builder, src: Value, scale: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.scaling_truncf(self.ctx, src.inner, scale.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + + // ==================== tt.map_elementwise / elementwise_inline_asm ==================== + + /// `tt.map_elementwise` — body builder receives one scalar Value per + /// `srcs` tensor (repeated `pack` times, flattened), and must return the + /// scalar results to feed back via `tt.map_elementwise.return`. + pub fn mapElementwise( + self: *Builder, + srcs: []const Value, + pack: i32, + scalar_arg_types: []const *const mlir.Type, + result_tensor_types: []const *const mlir.Type, + comptime body_fn: anytype, + body_ctx: anytype, + ) []Value { + const scratch = self.arena.allocator(); + const locs = scratch.alloc(*const mlir.Location, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (locs) |*l| l.* = self.loc(); + const body_block = mlir.Block.init(scalar_arg_types, locs); + + self.pushBlock(body_block); + const block_args = scratch.alloc(Value, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (block_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const out_vals: []const Value = body_fn(self, block_args, body_ctx); + _ = ttir.map_elementwise_return(self.ctx, self.innerSlice(out_vals), self.loc()).appendTo(body_block); + self.popBlock(); + + const op = ttir.map_elementwise( + self.ctx, + self.innerSlice(srcs), + pack, + body_block, + result_tensor_types, + self.loc(), + ); + return self.emitMulti(op, result_tensor_types.len); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype)` — inline asm + /// pointwise op with default options. Use `inlineAsmElementwiseOpts` for + /// `is_pure` / `pack`. + pub fn inlineAsmElementwise( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + ) []Value { + return self.inlineAsmElementwiseOpts(asm_string, constraints, args, result_types, .{}); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype, is_pure=..., pack=...)` + /// with the full named-param set. + pub fn inlineAsmElementwiseOpts( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + opts: InlineAsmOpts, + ) []Value { + const op = ttir.elementwise_inline_asm(self.ctx, asm_string, constraints, self.innerSlice(args), result_types, opts.is_pure, opts.pack, self.loc()); + return self.emitMulti(op, result_types.len); + } + + // ==================== scf.parallel / scf.reduce / scf.forall ==================== + + pub fn parallel( + self: *Builder, + lbs: []const Value, + ubs: []const Value, + steps: []const Value, + inits: []const Value, + reduction_result_types: []const *const mlir.Type, + comptime body_fn: anytype, + comptime combiners: anytype, + body_ctx: anytype, + ) []Value { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(inits.len == reduction_result_types.len); + + const scratch = self.arena.allocator(); + const idx_ty = .index(self.ctx); + const iv_types = scratch.alloc(*const mlir.Type, lbs.len) catch @panic("Builder.parallel OOM"); + const iv_locs = scratch.alloc(*const mlir.Location, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_types, iv_locs) |*t, *l| { + t.* = idx_ty; + l.* = self.loc(); + } + const body_block = mlir.Block.init(iv_types, iv_locs); + + self.pushBlock(body_block); + const iv_args = scratch.alloc(Value, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const reduce_operands: []const Value = body_fn(self, iv_args, body_ctx); + std.debug.assert(reduce_operands.len == reduction_result_types.len); + + // Build scf.reduce with one region per reduction. + const red_blocks = scratch.alloc(*mlir.Block, combiners.len) catch @panic("Builder.parallel OOM"); + inline for (combiners, 0..) |comb, i| { + const arg_ty = reduce_operands[i].type_(); + const arg_locs: [2]*const mlir.Location = .{ self.loc(), self.loc() }; + const red_block = mlir.Block.init(&.{ arg_ty, arg_ty }, &arg_locs); + self.pushBlock(red_block); + const lhs_v: Value = .{ .inner = red_block.argument(0), .kernel = self }; + const rhs_v: Value = .{ .inner = red_block.argument(1), .kernel = self }; + const combined: Value = comb(self, lhs_v, rhs_v, body_ctx); + _ = scf.reduce_return(self.ctx, combined.inner, self.loc()).appendTo(red_block); + self.popBlock(); + red_blocks[i] = red_block; + } + + _ = scf.reduce(self.ctx, self.innerSlice(reduce_operands), red_blocks, self.loc()).appendTo(body_block); + self.popBlock(); + + const op = scf.parallel( + self.ctx, + self.innerSlice(lbs), + self.innerSlice(ubs), + self.innerSlice(steps), + self.innerSlice(inits), + body_block, + reduction_result_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + + const out = self.arena.allocator().alloc(Value, reduction_result_types.len) catch @panic("Builder.parallel OOM"); + for (0..reduction_result_types.len) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + // ==================== reduce / scan ==================== + + /// `tt.reduce` along `axis`, single-input. See `ReduceArgs` for fields. + /// `ctx` is separate from `args` so `ReduceArgs(@TypeOf(ctx))` can be + /// inferred. + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.reduce_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.reduce(self.ctx, &.{args.src.inner}, args.axis, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + /// `tt.scan` along `axis`, single-input. See `ScanArgs` for fields. + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.scan_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.scan(self.ctx, &.{args.src.inner}, args.axis, args.reverse, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + // ==================== SCF regions ==================== + + /// `for iv in [lower, upper) step` carrying `inits`. Literal bounds take the + /// type of a Value bound, else `lower` is lifted (`i32` when it fits). + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); + } + + pub fn returnIf(self: *Builder, cond: Value) void { + self.returnIfOpts(cond, .{}); + } + + pub fn returnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) void { + const current = self.currentBlock(); + const region = current.parentRegion(); + const cont_block = mlir.Block.init(&.{}, &.{}); + + const ret_target: *mlir.Block = if (opts.inline_return) blk: { + const rb = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(rb); + region.appendOwnedBlock(rb); + break :blk rb; + } else self.exitBlock(); + + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br(self.ctx, cond.inner, ret_target, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); + self.continueIn(cont_block); + } + + /// The block every non-inline return branches to, which `finish` appends. + fn exitBlock(self: *Builder) *mlir.Block { + if (self.exit_block == null) { + self.exit_block = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(self.exit_block.?); + } + return self.exit_block.?; + } + + /// Continues in `block`, the fall-through of an early return, in place of + /// the current block. + fn continueIn(self: *Builder, block: *mlir.Block) void { + self.block_stack.replaceTop(block); + } + + pub fn openReturnIf(self: *Builder, cond: Value) ReturnIfScope { + return self.openReturnIfOpts(cond, .{}); + } + + pub fn openReturnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) ReturnIfScope { + const current = self.currentBlock(); + const region = current.parentRegion(); + + const ret_block = mlir.Block.init(&.{}, &.{}); + const cont_block = mlir.Block.init(&.{}, &.{}); + region.appendOwnedBlock(ret_block); + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br(self.ctx, cond.inner, ret_block, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); + + self.pushBlock(ret_block); + + return .{ + .kernel = self, + .ret_block = ret_block, + .cont_block = cont_block, + .opts = opts, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== finalization ==================== + + /// Append the final `tt.return` (kernels return nothing), verify the + /// module, and serialize to a NUL-terminated TTIR string owned by the + /// kernel's allocator (passed to `init`). Caller must + /// `defer allocator.free(ir)`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + // After any `returnIf` call, the entry block already has a + // `cf.cond_br` terminator; the fall-through block sits on the stack + // as the new insertion point. `currentBlock()` returns that block + // (the entry block if no cf was emitted), so the final terminator + // goes to the correct "tail" block in either case. + const current = self.currentBlock(); + if (self.exit_block) |exit| { + const region = self.entry_block.?.parentRegion(); + region.appendOwnedBlock(exit); + _ = cf.br(self.ctx, exit, &.{}, self.loc()).appendTo(current); + } else { + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(current); + } + + try common.verify(self.module, "dsl/triton"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = true, + .debug_info_pretty_form = false, + .print_name_loc_as_prefix = true, + })}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; diff --git a/dsl/triton/dtype.zig b/dsl/triton/dtype.zig new file mode 100644 index 00000000..45018e67 --- /dev/null +++ b/dsl/triton/dtype.zig @@ -0,0 +1,74 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Triton DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 1f655b55..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,7 +5,7 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k); + return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); } pub fn main(init: std.process.Init) !void { @@ -22,18 +22,26 @@ pub fn main(init: std.process.Init) !void { // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( + .{ .m = .low_bandwidth, .n = .high_bandwidth }, + )); const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - const a: zml.Tensor = .init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype); - const b: zml.Tensor = .init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype); + const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); + const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled benchmark [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); + defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); }; defer exe.deinit(); @@ -57,14 +65,14 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - exe.call(exe_args, &exe_results); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module const run_start: std.Io.Timestamp = .now(io, .awake); - exe.call(exe_args, &exe_results); + exe.call(io, exe_args, &exe_results, .{}); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); @@ -76,11 +84,11 @@ pub fn main(init: std.process.Init) !void { const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {D} - {d:.3} GFLOP/s", .{ + log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {f} - {d:.3} GFLOP/s", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), - stdx.fmt.fmtDuration(elapsed), + elapsed, flops / 1_000_000_000, }); } @@ -91,18 +99,19 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const switch (shape.dtype()) { inline else => |v| { - const ZigType = v.toZigType(); + const ZigType = v.toPackedZigType(); switch (comptime v.class()) { .bool => unreachable, .integer => { - for (slice.items(ZigType)) |*e| e.* = random.int(ZigType); + for (slice.items(ZigType)) |*e| e.* = @bitCast(random.int(@Int(.unsigned, @bitSizeOf(ZigType)))); }, .float => { const value = random.float(f32); for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { f64, f32 => value, f16 => @floatCast(value), - inline else => |T| if (@hasDecl(T, "fromF32")) T.fromF32(value) else unreachable, + zml.floats.Float4E2M1.Packed => .fromF32(value, -value), + inline else => |T| if (@hasDecl(T, "fromF32")) .fromF32(value) else unreachable, }; }, .complex => unreachable, diff --git a/examples/vfs/BUILD.bazel b/examples/io/BUILD.bazel similarity index 68% rename from examples/vfs/BUILD.bazel rename to examples/io/BUILD.bazel index aa3a533f..d471f74d 100644 --- a/examples/vfs/BUILD.bazel +++ b/examples/io/BUILD.bazel @@ -5,11 +5,7 @@ zig_binary( name = "playground", args = [], data = [], - main = "playground.zig", + main = "main.zig", deps = ["//zml"], -) - -build_test( - name = "test", - targets = [":mnist"], + visibility = ["//visibility:public"], ) diff --git a/examples/vfs/playground.zig b/examples/io/main.zig similarity index 71% rename from examples/vfs/playground.zig rename to examples/io/main.zig index 64b2f084..c9dc05ea 100644 --- a/examples/vfs/playground.zig +++ b/examples/io/main.zig @@ -9,15 +9,16 @@ pub const std_options: std.Options = .{ .log_level = .info, }; +const Command = enum { cat, tree, ls, cp, stat, realpath, safetensors, load }; + // -- ls hf://openai/gpt-oss-20b@6cee5e8 // -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 // -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f +// -- tree s3://noaa-goes19/ABI-Flood-Day-Shapefiles/2025/08 // -- cat https://iprs.fly.dev pub fn main(init: std.process.Init) !void { const allocator = init.gpa; - const Command = enum { cat, ls, cp, stat, realpath, safetensors }; - var it = init.minimal.args.iterate(); _ = it.next(); // skip program name const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; @@ -40,6 +41,9 @@ pub fn main(init: std.process.Init) !void { var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); defer s3_vfs.deinit(); + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + var vfs: zml.io.VFS = try .init(allocator, init.io); defer vfs.deinit(); @@ -47,6 +51,7 @@ pub fn main(init: std.process.Init) !void { try vfs.register("https", vfs_https.io()); try vfs.register("hf", hf_vfs.io()); try vfs.register("s3", s3_vfs.io()); + try vfs.register("gs", gcs_vfs.io()); const io = vfs.io(); @@ -68,7 +73,7 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); }, - .ls => { + .ls, .tree => { var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); defer dir.close(io); @@ -76,7 +81,7 @@ pub fn main(init: std.process.Init) !void { try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); var counts: TreeCounts = .{}; - try printTree(io, &stdout_writer.interface, dir, "", 10, &counts); + try printTree(io, &stdout_writer.interface, dir, "", if (command == .tree) 10 else 1, &counts); try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); }, .cp => { @@ -127,15 +132,11 @@ pub fn main(init: std.process.Init) !void { const root = try TensorNode.init(init.arena.allocator(), ""); - var registry_it = registry.iterator(); - while (registry_it.next()) |kv| { - const name = kv.key_ptr.*; - const tensor = kv.value_ptr.*; - + for (registry.tensors.values()) |tensor| { var current = root; - var parts = std.mem.tokenizeScalar(u8, name, '.'); + var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); while (parts.next()) |part| { - const gop = try current.children.getOrPut(part); + const gop = try current.children.getOrPut(current.allocator, part); if (!gop.found_existing) { gop.value_ptr.* = try TensorNode.init(init.arena.allocator(), part); } @@ -149,6 +150,76 @@ pub fn main(init: std.process.Init) !void { try printTensorTree(&stdout_writer.interface, root, "", true, true); try stdout_writer.interface.flush(); }, + .load => { + const MeshType = enum { replicated, sharded }; + + const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const tensor_parallel: *const zml.Mesh = try platform.registerMesh( + "tp", + .mesh(.{ .model = .high_bandwidth }), + ); + const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); + defer store.deinit(); + + const AllTensorsModel = struct { + tensors: []zml.Tensor, + }; + + const tensor_count = registry.tensors.count(); + + const tensors = try allocator.alloc(zml.Tensor, tensor_count); + defer allocator.free(tensors); + + var load_count: usize = 0; + for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(name, null), + .sharded => if (tensor.shape.rank() > 0) + store.view().createTensor(name, null, .tp, tp_partitioning) + else + store.view().createReplicatedTensor(name, null), + }; + load_count += 1; + } + + const model: AllTensorsModel = .{ .tensors = tensors }; + + var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); + progress.increaseEstimatedTotalItems(load_count); + defer progress.end(); + + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); + defer { + for (buffers.tensors) |*b| b.deinit(); + allocator.free(buffers.tensors); + } + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .parallelism = 8, + .dma_chunks = 16, + .dma_chunk_size = 256 * zml.MiB, + }); + defer loader.deinit(); + + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + }, } } @@ -160,7 +231,7 @@ const TreeCounts = struct { fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { if (max_depth == 0) return; - var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .{}; + var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .empty; var it = dir.iterate(); while (try it.next(io)) |entry| { entries.appendAssumeCapacity(entry); @@ -172,7 +243,7 @@ fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []cons const extension = if (is_last) " " else "│ "; const size: u64 = switch (entry.kind) { - .file => blk: { + .file, .sym_link => blk: { const stat = try dir.statFile(io, entry.name, .{}); break :blk stat.size; }, @@ -204,15 +275,15 @@ fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []cons const TensorNode = struct { name: []const u8, - children: std.StringArrayHashMap(*TensorNode), + children: std.StringArrayHashMapUnmanaged(*TensorNode) = .empty, + allocator: std.mem.Allocator, tensor: ?zml.safetensors.Tensor = null, fn init(allocator: std.mem.Allocator, name: []const u8) !*TensorNode { const node = try allocator.create(TensorNode); node.* = .{ .name = name, - .children = std.StringArrayHashMap(*TensorNode).init(allocator), - .tensor = null, + .allocator = allocator, }; return node; } @@ -244,7 +315,7 @@ fn printTensorTree( is_last: bool, is_root: bool, ) !void { - const allocator = node.children.allocator; + const allocator = node.allocator; if (is_root) { const children_nodes = try getSortedChildren(allocator, node); diff --git a/examples/lfm/BUILD.bazel b/examples/lfm/BUILD.bazel deleted file mode 100644 index a49cea2a..00000000 --- a/examples/lfm/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "lfm", - srcs = ["model.zig", "config.zig", "inference.zig", "discussion.zig", "test_runner.zig"], - main = "main.zig", - visibility = ["//visibility:public"], - deps = ["//zml", "@linenoise"], -) diff --git a/examples/lfm/config.zig b/examples/lfm/config.zig deleted file mode 100644 index 82ea9762..00000000 --- a/examples/lfm/config.zig +++ /dev/null @@ -1,83 +0,0 @@ -/// Copied from https://github.com/huggingface/transformers/blob/0997c2f2ab08c32c8e2f90aaad06e29a7108535b/src/transformers/models/lfm2/configuration_lfm2.py -/// -/// Configuration parsed from LFM2-style `config.json`. -/// Notes: -/// - `layer_types` is expected to be fully specified (HF can derive it from `full_attn_idxs`). -/// - HF may use `block_ff_dim` as the effective MLP size; keep `intermediate_size` and `block_ff_dim` -/// in sync in the JSON if you rely on both. -/// - HF uses `tie_embedding` as an alias for `tie_word_embeddings`. -/// - `eos_token_id` can differ from the HF default (e.g. 7 in LFM2 configs). -pub const Config = struct { - /// Model class names in the source config. - architectures: []const []const u8, - /// Whether to auto-adjust the FFN dimension. - block_auto_adjust_ff_dim: bool, - /// Base block hidden dimension. - block_dim: u32, - /// Block FFN dimension (HF uses this as the effective `intermediate_size` when present). - block_ff_dim: u32, - /// Multiplier for FFN dimension. - block_ffn_dim_multiplier: f32, - /// MLP init scale in block config. - block_mlp_init_scale: f32, - /// Multiple for intermediate size rounding. - block_multiple_of: u32, - /// RMS norm epsilon within blocks. - block_norm_eps: f32, - /// Output init scale in block config. - block_out_init_scale: f32, - /// Whether to use SwiGLU in blocks. - block_use_swiglu: bool, - /// Whether to use Xavier init in blocks. - block_use_xavier_init: bool, - /// BOS token id. - bos_token_id: u32, - /// Convolution cache length. - conv_L_cache: u32, - /// Whether conv layers use bias. - conv_bias: bool, - /// Convolution dimension. - conv_dim: u32, - /// Whether to use Xavier init in conv layers. - conv_use_xavier_init: bool, - /// Serialized dtype (e.g. "bfloat16"). - dtype: []const u8, - /// EOS token id. - eos_token_id: u32, - /// Hidden size of the model. - hidden_size: u32, - /// Weight init stddev. - initializer_range: f32, - /// FFN hidden dimension (HF may override with `block_ff_dim`). - intermediate_size: u32, - /// Per-layer operator types (e.g. "conv" or "full_attention"). - layer_types: []const []const u8, - /// Max sequence length supported by RoPE. - max_position_embeddings: u32, - /// Model type tag, expected "lfm2". - model_type: []const u8, - /// RMS norm epsilon. - norm_eps: f32, - /// Number of attention heads. - num_attention_heads: u32, - /// Total number of heads (may duplicate `num_attention_heads`). - num_heads: u32, - /// Number of transformer layers. - num_hidden_layers: u32, - /// Number of key/value heads for GQA/MQA. - num_key_value_heads: u32, - /// PAD token id. - pad_token_id: u32, - /// RoPE theta. - rope_theta: f32, - /// Whether to tie embeddings (aka `tie_word_embeddings`). - tie_embedding: bool, - /// Transformers version string. - transformers_version: []const u8, - /// Whether to use KV cache. - use_cache: bool, - /// Whether to use positional encodings. - use_pos_enc: bool, - /// Vocabulary size. - vocab_size: u32, -}; diff --git a/examples/lfm/discussion.zig b/examples/lfm/discussion.zig deleted file mode 100644 index 7f6fcd81..00000000 --- a/examples/lfm/discussion.zig +++ /dev/null @@ -1,296 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); -const stdx = zml.stdx; -const attention = zml.attention.attention; - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -const log = std.log.scoped(.lfm); - -pub const DEFAULT_PROMPT: [:0]const u8 = "> "; - -pub const Context = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), - exe: inference.Inference, - tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, - seqlen: u32, - cache_buffers: zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(attention.Metadata), - rng_buf: zml.Bufferized(zml.Tensor.Rng), - generated_token_slice: zml.Slice, - token_pos: u32, - all_tokens: std.ArrayList(u32), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), - cache: model.Cache, - attention_metadata: attention.Metadata, - exe: inference.Inference, - tokenizer: zml.tokenizer.Tokenizer, - config: model.Config, - seqlen: u32, - ) !Context { - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .model_buffers = model_buffers, - .exe = exe, - .tokenizer = tokenizer, - .config = config, - .seqlen = seqlen, - .cache_buffers = try cache.initBuffers(allocator, io, platform), - .attention_metadata_buffers = try attention_metadata.initBuffer(io, platform), - .rng_buf = try zml.Tensor.Rng.initBuffer(platform, seed, io), - .generated_token_slice = try .alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)), - .token_pos = 0, - .all_tokens = try .initCapacity(allocator, seqlen), - }; - } - - pub fn deinit(self: *Context) void { - model.Cache.unloadBuffers(&self.cache_buffers); - attention.Metadata.deinitBuffer(&self.attention_metadata_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buf); - self.generated_token_slice.free(self.allocator); - self.all_tokens.deinit(self.allocator); - } - - pub fn runOnce(self: *Context, prompt: []const u8) !void { - const prompt_tokens: []const u32 = try tokenizePrompt( - self.allocator, - self.tokenizer, - self.config, - prompt, - ); - defer self.allocator.free(prompt_tokens); - - var stdout = std.Io.File.stdout().writer(self.io, &.{}); - - log.info("Running prefill...", .{}); - try self.runPrefill(prompt_tokens); - - log.info("Running decode...", .{}); - try self.runDecode(&stdout.interface); - } - - pub fn start(self: *Context, prompt: []const u8) !void { - var turn_tokens: []const u32 = try tokenizePrompt( - self.allocator, - self.tokenizer, - self.config, - prompt, - ); - defer self.allocator.free(turn_tokens); - - var stdout = std.Io.File.stdout().writer(self.io, &.{}); - - while (self.remainingTokens() > 0) { - try stdout.interface.writeAll("\x1b[2mprefill...\x1b[0m"); - try stdout.interface.flush(); - - const prefill_start: std.Io.Timestamp = .now(self.io, .awake); - const prefill_pos_before = self.token_pos; - try self.runPrefill(turn_tokens); - const prefill_duration = prefill_start.untilNow(self.io, .awake); - const prefill_tokens: u64 = self.token_pos - prefill_pos_before; - - try stdout.interface.writeAll("\r \r"); - try stdout.interface.flush(); - - const decode_start: std.Io.Timestamp = .now(self.io, .awake); - const decode_pos_before = self.token_pos; - try self.runDecode(&stdout.interface); - const decode_duration = decode_start.untilNow(self.io, .awake); - const decode_tokens: u64 = self.token_pos - decode_pos_before; - - try stdout.interface.writeAll("\n\n"); - - try stdout.interface.print("\x1b[36mprefill\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m \x1b[2m\xe2\x94\x82\x1b[0m \x1b[36mdecode\x1b[0m \x1b[2m{D} \xc2\xb7 {:.1}tok/s\x1b[0m\n\n\n", .{ - stdx.fmt.fmtDuration(prefill_duration), - tokensPerSecond(prefill_duration, prefill_tokens), - stdx.fmt.fmtDuration(decode_duration), - tokensPerSecond(decode_duration, decode_tokens), - }); - try stdout.interface.flush(); - - while (true) { - const line = c.linenoise(DEFAULT_PROMPT.ptr) orelse { - try stdout.interface.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.token_pos, self.seqlen }); - try stdout.interface.flush(); - return; - }; - defer c.linenoiseFree(line); - const input = std.mem.sliceTo(line, 0); - if (input.len == 0) continue; - - self.allocator.free(turn_tokens); - turn_tokens = try tokenizeTurn(self.allocator, self.tokenizer, input); - break; - } - - if (turn_tokens.len > self.remainingTokens()) { - log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ self.remainingTokens(), turn_tokens.len, std.math.ceilPowerOfTwo(u32, self.seqlen + 1) catch std.math.maxInt(u32) }); - break; - } - } - } - - fn remainingTokens(self: Context) u32 { - return self.seqlen -| self.token_pos; - } - - fn runPrefill(self: *Context, prompt_tokens: []const u32) !void { - try self.all_tokens.appendSlice(self.allocator, prompt_tokens); - - if (self.all_tokens.items.len > self.seqlen) { - return error.PromptTooLong; - } - - const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); - defer tokens_slice.free(self.allocator); - const tokens = tokens_slice.items(u32); - @memset(tokens, self.config.pad_token_id); - @memcpy(tokens[0..self.all_tokens.items.len], self.all_tokens.items); - - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); - defer tokens_buf.deinit(); - - // Always prefill from position 0 with all accumulated tokens. - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); - defer tokens_pos_buf.deinit(); - - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(self.all_tokens.items.len)})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); - defer actual_seq_len_buf.deinit(); - - try self.exe.prefill.run(.{ - .allocator = self.allocator, - .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, - .tokens_buf = &tokens_buf, - .tokens_pos_buf = &tokens_pos_buf, - .actual_seq_len_buf = &actual_seq_len_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, - }); - - try tokens_buf.toSlice(self.io, tokens_slice); - self.generated_token_slice.items(u32)[0] = tokens_slice.items(u32)[self.all_tokens.items.len - 1]; - self.token_pos = @intCast(self.all_tokens.items.len); - } - - fn runDecode(self: *Context, writer: *std.Io.Writer) !void { - var tokenizer_decoder = try self.tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); - defer current_token_buffer.deinit(); - - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); - defer actual_seq_len_buf.deinit(); - - const think_start = self.tokenizer.tokenToId("") orelse unreachable; - const think_end = self.tokenizer.tokenToId("") orelse unreachable; - - generation: while (true) { - const generated_token = self.generated_token_slice.items(u32)[0]; - - if (generated_token == self.config.eos_token_id) break :generation; - if (try tokenizer_decoder.next(generated_token)) |chunk| { - if (generated_token == think_start) { - try writer.writeAll("\x1b[2m"); - } - try writer.writeAll(chunk); - if (generated_token == think_end) { - try writer.writeAll("\x1b[0m"); - } - try writer.flush(); - } - - // Accumulate the generated token. - try self.all_tokens.append(self.allocator, generated_token); - - if (self.remainingTokens() == 0) break :generation; - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{self.token_pos})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); - defer token_pos_buffer.deinit(); - - try self.exe.decode.run(.{ - .allocator = self.allocator, - .io = self.io, - .platform = self.platform, - .model_buffers = self.model_buffers, - .tokens_buf = ¤t_token_buffer, - .tokens_pos_buf = &token_pos_buffer, - .actual_seq_len_buf = &actual_seq_len_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = self.attention_metadata_buffers, - }); - - try current_token_buffer.toSlice(self.io, self.generated_token_slice); - self.token_pos += 1; - } - } -}; - -fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { - if (tokens == 0) return 0; - const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; - if (seconds <= 0) return 0; - return @as(f64, @floatFromInt(tokens)) / seconds; -} - -fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: model.Config, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); -} - -fn tokenizeTurn(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start, user, newline }); - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ im_end, newline }); - try tokens.appendSlice(allocator, &.{ im_start, assistant, newline }); - - return tokens.toOwnedSlice(allocator); -} diff --git a/examples/lfm/inference.zig b/examples/lfm/inference.zig deleted file mode 100644 index 9feabfaf..00000000 --- a/examples/lfm/inference.zig +++ /dev/null @@ -1,382 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; -const attention = zml.attention.attention; - -const model = @import("model.zig"); - -const log = std.log.scoped(.lfm); - -pub const CompilationOptions = struct { - hidden_dim: usize, - batch_dim: usize, - single: bool, - rng: zml.Tensor.Rng, - cache: model.Cache, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, - - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: attention.Backend, single: bool) CompilationOptions { - stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); - const cache: model.Cache = .{ - .kv = .init(.init(.{ - .layer = mdl.num_attention_layers, - .batch = 1, - .k = seqlen, - .h = config.num_key_value_heads, - .hd = config.hidden_size / config.num_attention_heads, - }, mdl.embed_tokens.weight.dtype())), - .conv = .init(.init(.{ - .layer = mdl.num_conv_layers, - .batch = 1, - .seq = config.conv_L_cache, - .d = config.hidden_size, - }, mdl.embed_tokens.weight.dtype())), - }; - - return .{ - .single = single, - .hidden_dim = config.hidden_size, - .batch_dim = 1, - .rng = .init(), - .cache = cache, - .attention_metadata = .init(.fromBackend(backend, seqlen)), - .attention_parameters = .init(.fromBackend(backend)), - }; - } -}; - -pub const Args = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - model_buffers: *zml.Bufferized(model.Model), - tokens_buf: *zml.Buffer, - tokens_pos_buf: *zml.Buffer, - actual_seq_len_buf: *zml.Buffer, - rng_buf: *zml.Bufferized(zml.Tensor.Rng), - cache_buffers: *zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(attention.Metadata), -}; - -pub const Inference = struct { - prefill: KernelExe, - decode: KernelExe, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - progress: *std.Progress.Node, - ) !Inference { - if (opts.single) { - const prefill_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); - const decode_exe = try compileSingleKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); - - return .{ - .prefill = .{ .single = prefill_exe }, - .decode = .{ .single = decode_exe }, - }; - } else { - const prefill_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, seqlen, true, progress); - const decode_exe = try compileComposedKernelExe(allocator, io, platform, mdl, opts, 1, false, progress); - return .{ - .prefill = .{ .composed = prefill_exe }, - .decode = .{ .composed = decode_exe }, - }; - } - } - - pub fn deinit(self: Inference) void { - self.prefill.deinit(); - self.decode.deinit(); - } -}; - -pub const KernelExe = union(enum) { - single: SingleKernelExe, - composed: ComposedKernelExe, - - pub fn deinit(self: KernelExe) void { - switch (self) { - .single => |exe| exe.deinit(), - .composed => |exe| exe.deinit(), - } - } - - pub fn run(self: *const KernelExe, args: Args) !void { - switch (self.*) { - .single => |*exe| try exe.run(args), - .composed => |*exe| try exe.run(args), - } - } -}; - -pub const SingleKernelExe = struct { - exe: zml.Exe, - - fn deinit(self: SingleKernelExe) void { - self.exe.deinit(); - } - - pub fn run(self: *const SingleKernelExe, args: Args) !void { - _ = args.io; // autofix - _ = args.platform; // autofix - var exe_args = try self.exe.args(args.allocator); - defer exe_args.deinit(args.allocator); - - var results = try self.exe.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ - args.model_buffers, - args.tokens_buf, - args.tokens_pos_buf, - args.actual_seq_len_buf, - args.rng_buf, - args.cache_buffers, - args.attention_metadata_buffers, - }); - self.exe.call(exe_args, &results); - - var new_tokens, var new_cache, var new_rng = results.get(struct { - zml.Buffer, - zml.Bufferized(model.Cache), - zml.Bufferized(zml.Tensor.Rng), - }); - replaceBuffer(args.tokens_buf, &new_tokens); - replaceCacheBuffers(args.cache_buffers, &new_cache); - replaceBuffer(&args.rng_buf._state, &new_rng._state); - } -}; - -fn compileSingleKernelExe(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !SingleKernelExe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling single kernel...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled single kernel [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - - const exe = try platform.compile(allocator, io, mdl, .forward, .{ tokens, token_position_offset, actual_seq_len, opts.rng, opts.cache, opts.attention_metadata, opts.attention_parameters, model.ConvParameters{ .is_prefill = is_prefill } }); - - return .{ .exe = exe }; -} - -pub const ComposedKernelExe = struct { - embed_tokens: zml.Exe, - conv_layer: zml.Exe, - attn_layer: zml.Exe, - lm_head: zml.Exe, - - fn deinit(self: ComposedKernelExe) void { - self.embed_tokens.deinit(); - self.conv_layer.deinit(); - self.attn_layer.deinit(); - self.lm_head.deinit(); - } - - /// Orchestrates the 4 sub-executables to replicate Model.forward behavior. - /// - /// Inputs and outputs mirror the monolithic Model.forward signature: - /// - Inputs: model_buffers, tokens, position_offset, actual_seq_len, rng, cache, attention_metadata - /// - Outputs: tokens (mutated), cache (mutated), rng (mutated) - pub fn run(self: *const ComposedKernelExe, args: Args) !void { - // Step 1: embed_tokens — inputs: [weight, tokens] → output: [hidden] - var hidden_buf: zml.Buffer = b: { - var exe_args = try self.embed_tokens.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try self.embed_tokens.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ args.model_buffers.embed_tokens, args.tokens_buf }); - self.embed_tokens.call(exe_args, &results); - break :b results.get(zml.Buffer); - }; - defer hidden_buf.deinit(); - - // Step 2: iterate over layers, dispatching to conv_layer or attn_layer exe. - var conv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); - defer conv_cache_index_buf.deinit(); - var kv_cache_index_buf: zml.Buffer = try .scalar(args.io, args.platform, @as(u32, 0), .u32); - defer kv_cache_index_buf.deinit(); - - for (args.model_buffers.layers) |layer_bufs| { - const exe = switch (layer_bufs.operator) { - .conv => &self.conv_layer, - .self_attn => &self.attn_layer, - }; - - var exe_args = try exe.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try exe.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ layer_bufs, &hidden_buf, args.tokens_pos_buf, args.actual_seq_len_buf, args.cache_buffers, &conv_cache_index_buf, &kv_cache_index_buf, args.attention_metadata_buffers }); - exe.call(exe_args, &results); - - // Outputs: { hidden, cache, conv_cache_index, kv_cache_index } - var new_hidden, var new_cache, var new_conv_cache_index, var new_kv_cache_index = results.get(struct { - zml.Buffer, - zml.Bufferized(model.Cache), - zml.Buffer, - zml.Buffer, - }); - replaceBuffer(&hidden_buf, &new_hidden); - replaceCacheBuffers(args.cache_buffers, &new_cache); - replaceBuffer(&conv_cache_index_buf, &new_conv_cache_index); - replaceBuffer(&kv_cache_index_buf, &new_kv_cache_index); - } - - // Step 3: lm_head — inputs: [lm_head buffers, hidden, embed_tokens.weight, tokens, rng] → outputs: [tokens, rng] - { - var exe_args = try self.lm_head.args(args.allocator); - defer exe_args.deinit(args.allocator); - var results = try self.lm_head.results(args.allocator); - defer results.deinit(args.allocator); - - exe_args.set(.{ args.model_buffers.lm_head, hidden_buf, args.model_buffers.embed_tokens, args.tokens_buf, args.rng_buf }); - self.lm_head.call(exe_args, &results); - var new_tokens, var new_rng = results.get(struct { zml.Buffer, zml.Bufferized(zml.Tensor.Rng) }); - replaceBuffer(args.tokens_buf, &new_tokens); - replaceBuffer(&args.rng_buf._state, &new_rng._state); - } - } -}; - -fn replaceCacheBuffers(dst: *zml.Bufferized(model.Cache), src: *zml.Bufferized(model.Cache)) void { - replaceBuffer(&dst.conv.state, &src.conv.state); - replaceBuffer(&dst.kv.k, &src.kv.k); - replaceBuffer(&dst.kv.v, &src.kv.v); -} - -fn replaceBuffer(dst: *zml.Buffer, src: *zml.Buffer) void { - if (!sameBufferHandle(dst.*, src.*)) { - dst.deinit(); - } - dst.* = src.*; -} - -fn sameBufferHandle(a: zml.Buffer, b: zml.Buffer) bool { - if (a._shards.len != b._shards.len) return false; - for (a._shards.constSlice(), b._shards.constSlice()) |a_shard, b_shard| { - if (a_shard != b_shard) return false; - } - return true; -} - -pub fn compileComposedKernelExe( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - is_prefill: bool, - progress: *std.Progress.Node, -) !ComposedKernelExe { - const embed_tokens_exe = try compileEmbedTokens(allocator, io, platform, mdl.embed_tokens, opts, seqlen, progress); - const conv_layer_exe = try compileConvLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); - const attn_layer_exe = try compileAttnLayer(allocator, io, platform, mdl, opts, seqlen, is_prefill, progress); - const lm_head_exe = try compileLmHead(allocator, io, platform, mdl, opts, seqlen, progress); - - return .{ - .embed_tokens = embed_tokens_exe, - .conv_layer = conv_layer_exe, - .attn_layer = attn_layer_exe, - .lm_head = lm_head_exe, - }; -} - -fn compileEmbedTokens(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, embed_tokens: model.TokenEmbedding, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling embed_tokens...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled embed_tokens [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - return platform.compile(allocator, io, embed_tokens, .forward, .{tokens}); -} - -fn compileConvLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling conv layer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled conv layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - // Find the first conv layer as a representative for compilation. - const conv_layer = for (mdl.layers) |layer| { - if (layer.operator == .conv) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return platform.compile(allocator, io, conv_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = is_prefill }, - }); -} - -fn compileAttnLayer(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, is_prefill: bool, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling attn layer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled attn layer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - // Find the first attention layer as a representative for compilation. - const attn_layer = for (mdl.layers) |layer| { - if (layer.operator == .self_attn) break layer; - } else unreachable; - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const token_position_offset: zml.Tensor = .init(.{ .batch = opts.batch_dim }, .u32); - const actual_seq_len: zml.Tensor = .init(.{}, .u32); - const conv_cache_index: zml.Tensor = .init(.{}, .u32); - const kv_cache_index: zml.Tensor = .init(.{}, .u32); - - return platform.compile(allocator, io, attn_layer, .forward, .{ - hidden, - token_position_offset, - actual_seq_len, - opts.cache, - conv_cache_index, - kv_cache_index, - opts.attention_metadata, - opts.attention_parameters, - model.ConvParameters{ .is_prefill = is_prefill }, - }); -} - -fn compileLmHead(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, mdl: model.Model, opts: CompilationOptions, seqlen: u32, progress: *std.Progress.Node) !zml.Exe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Compiling lm_head...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled lm_head [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - const hidden: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()); - const tokens: zml.Tensor = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32); - - return platform.compile(allocator, io, mdl.lm_head, .forward, .{ hidden, mdl.embed_tokens, tokens, opts.rng }); -} diff --git a/examples/lfm/main.zig b/examples/lfm/main.zig deleted file mode 100644 index 6f93fb01..00000000 --- a/examples/lfm/main.zig +++ /dev/null @@ -1,172 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); -const attention = zml.attention.attention; -const stdx = zml.stdx; - -const discussion = @import("discussion.zig"); -const inference = @import("inference.zig"); -const model = @import("model.zig"); -const test_runner = @import("test_runner.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const log = std.log.scoped(.lfm); -const Gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }); - -const Args = struct { - model: []const u8, - prompt: ?[]const u8 = null, - seqlen: u32 = 2048, - backend: ?attention.Backend = null, - run_tests: bool = false, - single: bool = false, - - pub const help = - \\Use lfm --model= [options] - \\ - \\ Run LFM model inference. - \\ - \\ Options: - \\ --model= Path to the model repository (required) - \\ --prompt= Prompt to use for generation (default: none) - \\ --seqlen= Sequence length (default: 1024) - \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --run-tests Run tests instead of inference - \\ --single Create a single kernel encompassing all the layers. - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - - const platform: *zml.Platform = try .auto(allocator, io, .{ .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.8 } } } }); - defer platform.deinit(allocator); - - log.info("\n{f}", .{platform.fmtVerbose()}); - - const args = stdx.flags.parse(init.minimal.args, Args); - - log.info("Resolving model repository..", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - - log.info("Loading configuration...", .{}); - const parsed_config = try loadConfig(model.Config, allocator, io, repo); - defer parsed_config.deinit(); - const config = parsed_config.value; - - log.info("Initializing model...", .{}); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - var mdl: model.Model = .init(allocator, store.view(), config); - defer mdl.deinit(allocator); - - const attention_backend = args.backend orelse b: { - const selected = attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - errdefer progress.end(); - - var tokenizer_fut = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); - defer if (tokenizer_fut.cancel(io)) |tokenizer| { - var tk = tokenizer; - tk.deinit(); - } else |_| {}; - - const params: inference.CompilationOptions = .init(mdl, config, args.seqlen, attention_backend, args.single); - - var compilation_fut = try io.concurrent(inference.Inference.init, .{ allocator, io, platform, mdl, params, args.seqlen, &progress }); - defer if (compilation_fut.cancel(io)) |exe| { - exe.deinit(); - } else |_| {}; - - var load_model_fut = try io.concurrent(model.Model.loadBuffers, .{ - &mdl, - allocator, - io, - platform, - &store, - &progress, - }); - defer if (load_model_fut.cancel(io)) |model_buffers| { - var mb = model_buffers; - model.Model.unloadBuffers(&mb, allocator); - } else |_| {}; - - const exe: inference.Inference = try compilation_fut.await(io); - var model_buffers: zml.Bufferized(model.Model) = try load_model_fut.await(io); - const tokenizer = try tokenizer_fut.await(io); - progress.end(); - - if (args.run_tests) { - return test_runner.run( - allocator, - io, - platform, - config, - mdl, - &model_buffers, - &store, - params.attention_metadata, - params.attention_parameters, - ); - } - - const interactive = args.prompt == null; - const prompt = if (args.prompt) |p| p else blk: { - const line = c.linenoise(discussion.DEFAULT_PROMPT) orelse return; - defer c.linenoiseFree(line); - break :blk try allocator.dupe(u8, std.mem.sliceTo(line, 0)); - }; - defer allocator.free(prompt); - - var ctx = try discussion.Context.init(allocator, io, platform, &model_buffers, params.cache, params.attention_metadata, exe, tokenizer, config, args.seqlen); - defer ctx.deinit(); - - if (interactive) { - return ctx.start(prompt); - } else { - return ctx.runOnce(prompt); - } -} - -pub fn loadConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { - const file = try dir.openFile(io, "config.json", .{}); - defer file.close(io); - - var buffer: [256]u8 = undefined; - var file_reader = file.reader(io, &buffer); - var reader: std.json.Reader = .init(allocator, &file_reader.interface); - defer reader.deinit(); - - return try std.json.parseFromTokenSource(model.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); -} - -pub fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Loading tokenizer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} diff --git a/examples/lfm/model.zig b/examples/lfm/model.zig deleted file mode 100644 index c8925f0d..00000000 --- a/examples/lfm/model.zig +++ /dev/null @@ -1,634 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const attention = zml.attention.attention; - -const cfg = @import("config.zig"); -pub const Config = cfg.Config; - -const log = std.log.scoped(.lfm); - -pub const Model = struct { - embed_tokens: TokenEmbedding, - lm_head: LmHead, - layers: []DecoderLayer, - num_attention_layers: usize, - num_conv_layers: usize, - - pub fn init(allocator: std.mem.Allocator, root_store: zml.io.TensorStore.View, config: Config) Model { - const store = root_store.withPrefix("model"); - stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); - - const layers = allocator.alloc(DecoderLayer, config.num_hidden_layers) catch unreachable; - var num_attention_layers: usize = 0; - var num_conv_layers: usize = 0; - for (layers, 0..) |*layer, i| { - const layer_store = store.withPrefix("layers").withLayer(i); - const kind = DecoderLayer.parseOperatorKind(config.layer_types[i]); - switch (kind) { - .conv => num_conv_layers += 1, - .full_attention => num_attention_layers += 1, - } - layer.* = DecoderLayer.init(config, layer_store, kind); - } - - return .{ - .embed_tokens = .init(store.withPrefix("embed_tokens")), - .lm_head = LmHead.init(store, config), - .layers = layers, - .num_attention_layers = num_attention_layers, - .num_conv_layers = num_conv_layers, - }; - } - - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - - const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - - defer { - const took = now.untilNow(io, .awake); - const took_ns: usize = @max(1, @as(usize, @intCast(took.toNanoseconds()))); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ - total_bytes, - stdx.fmt.fmtDuration(took), - total_bytes * std.time.ns_per_s / took_ns, - }); - } - - return zml.io.load(Model, self, allocator, io, platform, .{ - .dma_chunks = 8, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - LmHead.unloadBuffers(&self.lm_head); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - - pub fn forward( - // - self: Model, - tokens: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, - rng: Tensor.Rng, - cache_: Cache, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, - conv_parameters: ConvParameters, - ) struct { Tensor, Cache, Tensor.Rng } { - stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); - - const embeds = self.embed_tokens.forward(tokens); - - var hidden = embeds; - var cache = cache_; - var conv_cache_index = Tensor.scalar(@as(u32, 0), .u32); - var kv_cache_index = Tensor.scalar(@as(u32, 0), .u32); - for (self.layers) |layer| { - hidden, cache, conv_cache_index, kv_cache_index = layer.forward(hidden, tokens_position_offset, actual_seq_len, cache, conv_cache_index, kv_cache_index, attention_metadata, attention_parameters, conv_parameters); - } - - const new_tokens, const new_rng = self.lm_head.forward(hidden, self.embed_tokens, tokens, rng); - return .{ new_tokens, cache.reuseBuffer(cache_), new_rng }; - } - - pub fn deinit(self: Model, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } -}; - -pub const TokenEmbedding = struct { - weight: Tensor, - - pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight").withTags(.{ .voc, .d }) }; - } - - pub fn forward(self: TokenEmbedding, tokens: Tensor) Tensor { - stdx.debug.assert(tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{tokens}); - stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); - return self.weight.gather(.{ .voc = tokens }, .{}); - } - - pub fn unembed(self: TokenEmbedding, embeds: Tensor) Tensor { - stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); - return self.weight.dot(embeds, .d); - } -}; - -pub const LmHead = struct { - embedding_norm: RmsNorm, - - pub fn init(store: zml.io.TensorStore.View, config: Config) LmHead { - return .{ - .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), - }; - } - - pub fn forward(self: LmHead, hidden: Tensor, embed_tokens: TokenEmbedding, tokens: Tensor, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { - const logits = embed_tokens.unembed(self.embedding_norm.forward(hidden)); - const new_tokens, const new_rng = zml.nn.sampleTokens(logits, .{}, rng); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), new_rng }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { - RmsNorm.unloadBuffers(&self.embedding_norm); - } -}; - -pub const OperatorKind = enum { conv, full_attention }; -const Operator = union(enum) { - conv: ShortConv, - self_attn: Attention, -}; - -pub const DecoderLayer = struct { - operator: Operator, - operator_norm: RmsNorm, - ffn_norm: RmsNorm, - feed_forward: Mlp, - - pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { - return std.meta.stringToEnum(OperatorKind, layer_type) orelse { - stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); - unreachable; - }; - } - - pub fn init(config: Config, store: zml.io.TensorStore.View, kind: OperatorKind) DecoderLayer { - const operator_norm = RmsNorm.init(store.withPrefix("operator_norm"), config.norm_eps, .d); - const ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps, .d); - const feed_forward = Mlp.init(store.withPrefix("feed_forward")); - const operator: Operator = switch (kind) { - .conv => .{ .conv = ShortConv.init(config, store.withPrefix("conv")) }, - .full_attention => .{ .self_attn = Attention.init(config, store.withPrefix("self_attn")) }, - }; - - return .{ - .operator = operator, - .operator_norm = operator_norm, - .ffn_norm = ffn_norm, - .feed_forward = feed_forward, - }; - } - - pub fn forward( - self: DecoderLayer, - input: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, - cache_: Cache, - conv_cache_index_: Tensor, - kv_cache_index_: Tensor, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, - conv_parameters: ConvParameters, - ) struct { Tensor, Cache, Tensor, Tensor } { - var cache = cache_; - var conv_cache_index = conv_cache_index_; - var kv_cache_index = kv_cache_index_; - const residual = switch (self.operator) { - .conv => |operator| b: { - const residual, const updated_conv_cache = operator.forward( - self.operator_norm.forward(input), - tokens_position_offset, - actual_seq_len, - cache.conv, - conv_cache_index, - conv_parameters, - ); - cache.conv = updated_conv_cache; - conv_cache_index = conv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); - break :b residual; - }, - .self_attn => |operator| b: { - const residual, const updated_kv_cache = operator.forward( - self.operator_norm.forward(input), - tokens_position_offset, - cache.kv, - kv_cache_index, - attention_metadata, - attention_parameters, - ); - cache.kv = updated_kv_cache; - kv_cache_index = kv_cache_index.add(Tensor.scalar(@as(u32, 1), .u32)); - break :b residual; - }, - }; - const x = input.add(residual); - return .{ - x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input), - cache.reuseBuffer(cache_), - conv_cache_index.reuseBuffer(conv_cache_index_), - kv_cache_index.reuseBuffer(kv_cache_index_), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - switch (self.operator) { - .conv => |*operator| ShortConv.unloadBuffers(operator), - .self_attn => |*operator| Attention.unloadBuffers(operator), - } - RmsNorm.unloadBuffers(&self.operator_norm); - RmsNorm.unloadBuffers(&self.ffn_norm); - Mlp.unloadBuffers(&self.feed_forward); - } -}; - -pub const ConvParameters = struct { is_prefill: bool }; - -pub const ShortConv = struct { - in_proj: Linear, - out_proj: Linear, - kernel: zml.Tensor, - config: Config, - - pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { - stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); - return ShortConv{ .in_proj = initLinear(store.withPrefix("in_proj"), .d), .out_proj = initLinear(store.withPrefix("out_proj"), .d), .kernel = store.createTensorWithTags("conv.weight", .{ .out, .in, .kernel_size }), .config = config }; - } - - pub fn forward( - self: ShortConv, - input: Tensor, - tokens_position_offset: Tensor, - actual_seq_len: Tensor, - cache_: ConvCache, - cache_index: Tensor, - parameters: ConvParameters, - ) struct { Tensor, ConvCache } { - var cache = cache_; - // Allows us to have a consistent API for the generic DecoderLayer - const BCx = self.in_proj.forward(input); - - const B, const C, const x = BCx.chunkExact(.d, 3); - const Bx = B.mul(x); - - const conv_out = if (parameters.is_prefill) b: { - const actual_seq_len_i32 = actual_seq_len.convert(.i32); - const start = actual_seq_len_i32.sub(Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32)).maximum(Tensor.scalar(@as(i32, 0), .i32)); - const cache_seq_indices = lkp: { - const n = actual_seq_len_i32.sub(start); - const left_pad = Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); - const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); - break :lkp Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); - }; - const scatter_data = Bx.dynamicSlice(.{ .seq = @as(Tensor.DynSlice, .{ .start = start.convert(.u32), .len = @intCast(self.config.conv_L_cache) }) }); - cache.state = cache.state.scatterSlices( - .{ .layer = cache_index, .seq = cache_seq_indices }, - scatter_data, - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(cache.state); - const padding = self.config.conv_L_cache - 1; - const conv_out = Tensor.conv1d(Bx, self.kernel, .{ - .padding = &.{ padding, 0 }, - .input_batch_dimension = Bx.axis(.batch), - .input_feature_dimension = B.axis(.d), - .input_spatial_dimensions = B.axis(.seq), - .kernel_output_feature_dimension = self.kernel.axis(.out), - .kernel_input_feature_dimension = self.kernel.axis(.in), - .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), - .output_batch_dimension = Bx.axis(.batch), - .output_feature_dimension = Bx.axis(.d), - .output_spatial_dimensions = Bx.axis(.seq), - .feature_group_count = Bx.dim(.d), - }); - break :b conv_out.slice1d(.seq, .{ .end = Bx.dim(.seq) }); - } else b: { - cache.state = cache.state.rollRight1d(.seq, -1) - .scatterSlices( - .{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, @intCast(0)), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, - Bx, - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(cache.state); - const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); - const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); - - break :b cache.state.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer) - .mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })) - .sum(.seq); - }; - - const y = C.mul(conv_out); - - const output = self.out_proj.forward(y); - return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { - Linear.unloadBuffers(&self.in_proj); - Linear.unloadBuffers(&self.out_proj); - self.kernel.deinit(); - } -}; - -pub const Attention = struct { - q_proj: Linear, - k_proj: Linear, - v_proj: Linear, - out_proj: Linear, - q_layernorm: RmsNorm, - k_layernorm: RmsNorm, - - head_dim: usize, - num_key_value_groups: usize, - rope_opts: zml.nn.RopeOpts, - - pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { - const head_dim = config.hidden_size / config.num_attention_heads; - const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; - - return .{ - // - .q_proj = initLinear(store.withPrefix("q_proj"), .d), - .k_proj = initLinear(store.withPrefix("k_proj"), .d), - .v_proj = initLinear(store.withPrefix("v_proj"), .d), - .out_proj = initLinear(store.withPrefix("out_proj"), .d), - .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), - .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), - .head_dim = head_dim, - .num_key_value_groups = num_key_value_groups, - .rope_opts = .{ - .layout = .sequential, - .freq_base = config.rope_theta, - .scaling = .{ .default = .{} }, - }, - }; - } - - pub fn forward( - self: Attention, - x: Tensor, - tokens_position_offset: Tensor, - kv_cache: KvCache, - cache_index: Tensor, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, - ) struct { Tensor, KvCache } { - var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - - q = self.q_layernorm.forward(q); - k = self.k_layernorm.forward(k); - - const token_positions = b: { - const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); - break :b Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); - }; - - q = zml.nn.rope(q, token_positions, self.rope_opts); - k = zml.nn.rope(k, token_positions, self.rope_opts); - - q = q.rename(.{ .seq = .q }); - k = k.rename(.{ .seq = .k }); - v = v.rename(.{ .seq = .k }); - - const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); - k = new_kv_cache.keys(cache_index); - v = new_kv_cache.values(cache_index); - - stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); - const attn = switch (attention_parameters) { - .vanilla => attention.attention( - q, - k, - v, - tokens_position_offset, - attention_metadata, - attention_parameters, - ).merge(.{ .d = .{ .h, .hd } }), - // flash attention doesn't support batch dimension at all in practice. - else => attention.attention( - q.squeeze(.batch), - k.squeeze(.batch), - v.squeeze(.batch), - tokens_position_offset.squeeze(.batch), - attention_metadata, - attention_parameters, - ).merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .seq }).insertAxes(.seq, .{.batch}), - }; - - return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { - Linear.unloadBuffers(&self.q_proj); - Linear.unloadBuffers(&self.k_proj); - Linear.unloadBuffers(&self.v_proj); - Linear.unloadBuffers(&self.out_proj); - RmsNorm.unloadBuffers(&self.q_layernorm); - RmsNorm.unloadBuffers(&self.k_layernorm); - } -}; - -pub const Cache = struct { - conv: ConvCache, - kv: KvCache, - - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { - return .{ - .conv = try self.conv.initBuffers(allocator, io, platform), - .kv = try self.kv.initBuffers(io, platform), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { - ConvCache.unloadBuffers(&self.conv); - KvCache.unloadBuffers(&self.kv); - } - - pub fn reuseBuffer(self: Cache, other: Cache) Cache { - return .{ - .conv = self.conv.reuseBuffer(other.conv), - .kv = self.kv.reuseBuffer(other.kv), - }; - } -}; - -pub const ConvCache = struct { - state: Tensor, - - pub fn init(shape: zml.Shape) ConvCache { - return .{ - .state = .fromShape(shape), - }; - } - - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { - const sh = self.state.shape(); - const host = try allocator.alloc(u8, sh.byteSize()); - defer allocator.free(host); - @memset(host, 0); - - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { - self.state.deinit(); - } - - pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { - return .{ - .state = self.state.reuseBuffer(other.state), - }; - } -}; - -pub const KvCache = struct { - k: Tensor, - v: Tensor, - - pub fn init(kv_shape: zml.Shape) KvCache { - return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), - }; - } - - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - }; - } - - pub fn keys(self: KvCache, cache_index: Tensor) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); - } - - pub fn values(self: KvCache, cache_index: Tensor) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = cache_index, .len = 1 } }).squeeze(.layer); - } - - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_position: Tensor, cache_index: Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); - const layer = cache_index.broad(token_position.shape()); - - return .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .k = token_position }, - new_k.transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .k = token_position }, - new_v.transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - }; - } -}; - -fn initLinear(store: zml.io.TensorStore.View, tag: anytype) Linear { - return .init(store.createTensorWithTags("weight", .{ .out, tag }), null, tag); -} - -pub const Linear = struct { - weight: Tensor, - bias: ?Tensor = null, - in_tag: Shape.Tag, - out_tag: Shape.Tag, - - /// Assumes that the weights are tagged `.in` and `.out`. - pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { - stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); - const axis = weight.shape().axis(tag); - const out_tag = weight.shape().tag(1 - axis); - return .{ - .weight = weight, - .bias = bias, - .in_tag = zml.Shape.toTag(tag), - .out_tag = out_tag, - }; - } - - pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { - linear.weight.deinit(); - if (linear.bias) |*bias| bias.deinit(); - } - - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); - - return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; - } -}; - -const Mlp = struct { - w1: Linear, - w2: Linear, - w3: Linear, - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .w1 = initLinear(store.withPrefix("w1"), .d), - .w2 = initLinear(store.withPrefix("w2"), .d), - .w3 = initLinear(store.withPrefix("w3"), .d), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); - } - - pub fn forward(self: Mlp, x: Tensor) Tensor { - return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); - } -}; - -const RmsNorm = struct { - weight: Tensor, - eps: f32, - tag: Shape.Tag, - - pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createTensorWithTags("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { - const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); - return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); - } -}; diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel deleted file mode 100644 index c7c91eb4..00000000 --- a/examples/llama/BUILD.bazel +++ /dev/null @@ -1,80 +0,0 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@rules_zig//zig:defs.bzl", "zig_binary") - -zig_binary( - name = "llama", - srcs = ["llama.zig"], - main = "main.zig", - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "//zml", - ], -) - -zig_binary( - name = "loader", - srcs = ["llama.zig"], - main = "main2.zig", - visibility = ["//visibility:public"], - deps = ["//zml"], -) - -mtree_spec( - name = "mtree", - srcs = [":llama"], -) - -tar( - name = "archive", - srcs = [":llama"], - args = [ - "--options", - ",".join([ - "zstd:compression-level=9", - "zstd:threads=16", - ]), - ], - compress = "zstd", - mtree = ":mtree", -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["/{}/llama".format(package_name())], - target_compatible_with = [ - "@platforms//os:linux", - ], - tars = [":archive"], -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = ["zmlai/llama:latest"], - tags = ["manual"], -) - -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - repository = "index.docker.io/zmlai/llama", - tags = ["manual"], -) - -build_test( - name = "test", - targets = [":llama"], -) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig deleted file mode 100644 index 5073a140..00000000 --- a/examples/llama/llama.zig +++ /dev/null @@ -1,502 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; -const stdx = zml.stdx; - -const log = std.log.scoped(.llama); - -pub const TransferCtx = struct { - allocator: std.mem.Allocator, - pool: *zml.mem.DynamicBufferPool, - transferred_bytes: *usize, - progress: *std.Progress.Node, -}; - -/// Llama architecture, using huggingface transformers naming. -/// Dimensions of activations: {.b, .s, .d} -pub const LlamaLM = struct { - pub const Config = struct { - bos_token_id: u32, - eos_token_id: stdx.json.Union(union(enum) { - int: u32, - ints: []u32, - }), - head_dim: ?u32 = null, - hidden_size: u32, - num_hidden_layers: u32, - num_attention_heads: u32, - num_key_value_heads: u32, - rope_theta: f32, - max_position_embeddings: u32, - rms_norm_eps: f32, - hf_rope_impl: bool = true, - tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, - }; - - pub const Options = struct { - sampling_strategy: ?zml.nn.SamplingStrategy, - max_seq_len: u32, - }; - - lm_head: ?zml.nn.Linear, - model: Llama, - - // Options controlling generation - gen_opts: zml.nn.SamplingStrategy = .{}, - config: Config, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, options: Options) !LlamaLM { - const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d })) |weight| - .init(weight, null, .d) - else - null; - - return .{ - .lm_head = lm_head, - .model = try .init(allocator, store.withPrefix("model"), config), - .gen_opts = options.sampling_strategy orelse .{}, - .config = config, - }; - } - - pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { - self.model.deinit(allocator); - } - - pub fn load( - self: *const LlamaLM, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(LlamaLM) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - var total_bytes: usize = 0; - defer { - const took = now.untilNow(io, .awake); - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ total_bytes, stdx.fmt.fmtDuration(took), bytes_per_sec }); - } - return zml.io.load(LlamaLM, self, allocator, io, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 128 * zml.MiB, - .progress = progress, - .store = store, - .parallelism = 16, - .total_bytes = &total_bytes, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(LlamaLM), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| lm_head.weight.deinit(); - Llama.unloadBuffers(&self.model, allocator); - } - - /// Predicts the token at `token_index` position. - /// Returns: - /// - updated `tokens`, - /// - updated KV cache - /// - a Rng state to allow for probabilistic generation - pub fn forward( - self: LlamaLM, - tokens_: Tensor, - token_index: Tensor, - kv_cache: KvCache, - rng: Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache, Tensor.Rng } { - stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); - const tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); - const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); - return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; - } - - pub fn sampleTokens( - self: LlamaLM, - lm_head_: ?zml.nn.Linear, - out_: Tensor, - rng: Tensor.Rng, - opts: zml.nn.SamplingStrategy, - ) struct { Tensor, Tensor.Rng } { - const out = out_.withPartialTags(.{ .s, .d }); - - var logits = blk: { - if (lm_head_) |lm_head| { - break :blk lm_head.forward(out).rename(.{ .dout = .d }); - } else { - break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); - } - }; - - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - return .{ next_tokens, new_rng }; - } -}; - -pub const Llama = struct { - embed_tokens: zml.nn.TokenEmbedding, - norm: RmsNorm, - layers: []TransformerLayer, - - max_seq_len: u32 = 0, - num_heads: u32 = 32, - num_kv_heads: u32 = 32, - rope_opts: zml.nn.RopeOpts = .{ - .layout = .interleaved, - .freq_base = 10_000, - }, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config) !Llama { - const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); - errdefer allocator.free(layers); - - for (layers, 0..) |*layer, i| { - layer.* = try .init(store.withPrefix("layers").withLayer(i), config); - } - - return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight") }, - .norm = .{ .weight = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .eps = config.rms_norm_eps }, - .layers = layers, - }; - } - - pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - RmsNorm.unloadBuffers(&self.norm); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - - /// Forward one token, using KV cache for previous tokens. - /// Returns result and updated KV cache. - pub fn forward( - self: Llama, - tokens: Tensor, - token_index: Tensor, - kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { - const embeds = embed(self.embed_tokens, tokens); - var hidden = embeds; - - var updated_kv_cache = kv_cache; - for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i), attention_metadata, attention_parameters); - } - const output = self.norm.forward(hidden); - - return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; - } - - pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { - return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); - } -}; - -pub const TransformerLayer = struct { - input_layernorm: RmsNorm, - self_attn: SelfAttn, - post_attention_layernorm: RmsNorm, - mlp: Mlp, - - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !TransformerLayer { - return .{ - .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), - .self_attn = try .init(store.withPrefix("self_attn"), config), - .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), - .mlp = .init(store.withPrefix("mlp")), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - SelfAttn.unloadBuffers(&self.self_attn); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - Mlp.unloadBuffers(&self.mlp); - } - - pub fn forward( - self: TransformerLayer, - x0: Tensor, - token_index: Tensor, - kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { - // Self Attention - //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - - const x0_normalized = self.input_layernorm.forward(x0); - const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache, attention_metadata, attention_parameters); - const x1 = x0.add(delta0); - - // Fully Connected - const x1_normalized = self.post_attention_layernorm.forward(x1); - const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); - - return .{ x2.reuseBuffer(x0), updated_kv_cache }; - } -}; - -const RmsNorm = struct { - weight: Tensor, - eps: f32 = 1e-5, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensorWithTags("weight", .{.d}), .eps = eps }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { - const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - const normalized = zml.nn.rmsNorm(x, .d, self.eps); - return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); - } -}; - -const Mlp = struct { - up_proj: zml.nn.Linear, // (dim -> hidden_dim) - gate_proj: zml.nn.Linear, // (dim -> hidden_dim) - down_proj: zml.nn.Linear, // (hidden_dim -> dim) - - fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); - } - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .up_proj = initLinear(store.withPrefix("up_proj")), - .gate_proj = initLinear(store.withPrefix("gate_proj")), - .down_proj = initLinear(store.withPrefix("down_proj")), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - self.up_proj.weight.deinit(); - if (self.up_proj.bias) |*bias| bias.deinit(); - self.gate_proj.weight.deinit(); - if (self.gate_proj.bias) |*bias| bias.deinit(); - self.down_proj.weight.deinit(); - if (self.down_proj.bias) |*bias| bias.deinit(); - } - - pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = self.up_proj.forward(x); - var output = self.gate_proj.forward(x); - output = output.silu().mul(proj).rename(.{ .dout = .d }); - return self.down_proj.forward(output); - } -}; - -pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - - q_norm: ?RmsNorm, - k_norm: ?RmsNorm, - - o_proj: zml.nn.Linear, - num_heads: i64 = undefined, - num_kv_heads: i64 = 0, - rope_opts: zml.nn.RopeOpts = undefined, - - fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { - return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); - } - - pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { - return .{ - .q_proj = initProj(store.withPrefix("q_proj")), - .k_proj = initProj(store.withPrefix("k_proj")), - .v_proj = initProj(store.withPrefix("v_proj")), - .o_proj = initProj(store.withPrefix("o_proj")), - // TODO(Corentin): fix that - .q_norm = null, - .k_norm = null, - .num_heads = @intCast(config.num_attention_heads), - .num_kv_heads = @intCast(config.num_key_value_heads), - .rope_opts = .{ - .layout = if (config.hf_rope_impl) .sequential else .interleaved, - .freq_base = config.rope_theta, - .scaling = config.rope_scaling, - }, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - self.q_proj.weight.deinit(); - if (self.q_proj.bias) |*bias| bias.deinit(); - self.k_proj.weight.deinit(); - if (self.k_proj.bias) |*bias| bias.deinit(); - self.v_proj.weight.deinit(); - if (self.v_proj.bias) |*bias| bias.deinit(); - self.o_proj.weight.deinit(); - if (self.o_proj.bias) |*bias| bias.deinit(); - - if (self.q_norm) |*q_norm| RmsNorm.unloadBuffers(q_norm); - if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); - } - - /// Self Attention. - /// - If token_index is set, x is assumed to be the representation of one new token, - /// and kv_cache will be read for the previous tokens. - /// - If token_index is not set, x is assumed to be the representation of all tokens - /// since the beginning of the sequence, and kv_cache won't be read. - /// In both case, kv_cache will be updated with the computed key and value. - /// x: {.b, .s, .d } -> .{.b, .s, .d} - pub fn forward( - self: SelfAttn, - x: Tensor, - token_index: Tensor, - kv_cache: KvCache, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, - ) struct { Tensor, KvCache } { - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - - // In self-attention, .s axis is used both for keys and queries. - const pos_index = b: { - const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); - }; - - if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - q = zml.nn.rope(q, pos_index, self.rope_opts); - k = zml.nn.rope(k, pos_index, self.rope_opts); - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - const dtype = q.dtype(); - const new_kv_cache = kv_cache.update(k, v, token_index); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - - const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); - - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ self.o_proj.forward(attn), new_kv_cache }; - } -}; - -pub const KvCache = struct { - k: Tensor, - v: Tensor, - layer_index: Tensor, - - pub fn init(kv_shape: zml.Shape) KvCache { - return .{ - .k = .fromShape(kv_shape), - .v = .fromShape(kv_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { - return .{ - .k = kv_shape, - .v = kv_shape, - .layer_index = zml.Shape.init(.{}, .u32), - }; - } - - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - self.layer_index.deinit(); - } - - pub fn keys(self: KvCache) Tensor { - return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn values(self: KvCache) Tensor { - return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); - var layer = self.layer_index; - layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; - - return if (token_index) |idx| .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .k = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .k = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - } else .{ - .k = self.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: KvCache, layer_index: usize) KvCache { - return .{ - .k = self.k, - .v = self.v, - .layer_index = Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } -}; diff --git a/examples/llama/main.zig b/examples/llama/main.zig deleted file mode 100644 index e16ee303..00000000 --- a/examples/llama/main.zig +++ /dev/null @@ -1,478 +0,0 @@ -const std = @import("std"); -const log = std.log; - -const zml = @import("zml"); -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; -const stdx = zml.stdx; - -const llama = @import("llama.zig"); -const LlamaLM = llama.LlamaLM; -const Llama = llama.Llama; -const KvCache = llama.KvCache; -const TransformerLayer = llama.TransformerLayer; -const SelfAttn = llama.SelfAttn; - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const CliArgs = struct { - model: []const u8, - prompt: ?[]const u8 = null, - seqlen: u32 = 512, - reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, - writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, - async_limit: ?u32 = null, - backend: ?zml.attention.attention.Backend = null, - - pub const help = - \\Usage: llama --model= [options] - \\ - \\Run LLaMA inference on a model. - \\ - \\Options: - \\ --model= Path to model (local path or HuggingFace repo) - \\ --prompt= Input prompt (reads from stdin if not provided) - \\ --seqlen= Maximum sequence length (default: 512) - \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) - \\ --reader-buffer-size= Reader buffer size (default: 32MiB) - \\ --writer-buffer-size= Writer buffer size (default: 32MiB) - \\ --async-limit= Async I/O concurrency limit - \\ -h, --help Show this help message - \\ - \\Examples: - \\ llama --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Hello, world!" - \\ llama --model=$(realpath ../Llama-3.1-8B-Instruct) --seqlen=1024 - \\ echo "What is 2+2?" | llama --model=meta-llama/Llama-3.1-8B-Instruct - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const arena = init.arena; - - log.info("LLama was compiled with {}", .{@import("builtin").mode}); - - const args = stdx.flags.parse(init.minimal.args, CliArgs); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - - const io = vfs.io(); - - log.info("Resolving model repo", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - - const parsed_config = try parseConfig(allocator, io, repo); - defer parsed_config.deinit(); - const config = parsed_config.value; - - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - - // Write metadata from the config file into the LlamaLm struct. - const llama_options: llama.LlamaLM.Options = .{ - .max_seq_len = args.seqlen, - .sampling_strategy = .{ - .topk = 1, - .temperature = 1.0, - }, - }; - - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - log.info("\n{f}", .{platform.fmtVerbose()}); - - // Initialize the Llama struct and map the content of the .safetensors to the model tensors. - var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); - defer llama_model.deinit(allocator); - - const backend = args.backend orelse b: { - const selected = zml.attention.attention.Backend.auto(platform); - log.info("Selected backend: {}", .{selected}); - break :b selected; - }; - - // Specify shapes of input arguments - const dtype = llama_model.model.embed_tokens.weight.dtype(); - const llama_parameters: LlamaParameters = .{ - .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .u32), - .decode_tokens = .init(.{ .s = 1 }, .u32), - .token_index = .init(.{}, .u32), - .kv_cache = .init(.init(.{ - .layer = llama_model.model.layers.len, - .k = args.seqlen, - .h = config.num_key_value_heads, - .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), - }, dtype)), - .rng = .init(), - .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen))), - .attention_parameters = .init(.fromBackend(backend)), - }; - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - - var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); - errdefer blk: { - var v = tokenizer_future.cancel(io) catch break :blk; - v.deinit(); - } - - var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); - defer if (compiled_model_result_future.cancel(io)) |v| { - v.prefill_exe.deinit(); - v.decode_exe.deinit(); - } else |_| {}; - - var load_group: stdx.Io.LimitedGroup = .init(16); - defer load_group.cancel(io); - - var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ - &llama_model, - arena.allocator(), - io, - platform, - &store, - &progress, - }); - defer blk: { - var v = llama_buffers_future.cancel(io) catch break :blk; - LlamaLM.unloadBuffers(&v, arena.allocator()); - } - - const compiled_model_result = try compiled_model_result_future.await(io); - - var llama_buffers = try llama_buffers_future.await(io); - progress.end(); - - log.info("Creating KvCache", .{}); - var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); - defer llama.KvCache.deinitBuffer(&kv_cache_buffers); - - var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); - defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - const tokenizer = try tokenizer_future.await(io); - - const prompt = if (args.prompt) |p| p else blk: { - var reader = std.Io.File.stdin().reader(io, &.{}); - break :blk try reader.interface.allocRemaining(allocator, .unlimited); - }; - - log.info("✅ Prompt: {s}", .{prompt}); - - // Unbuffered writing of the tokens to stdout. - var stdout = std.Io.File.stdout().writer(io, &.{}); - - try generateText( - allocator, - io, - &llama_buffers, - compiled_model_result.prefill_exe, - compiled_model_result.decode_exe, - &kv_cache_buffers, - &attention_metadata_buffers, - tokenizer, - config, - llama_options, - @intCast(std.Io.Clock.now(.awake, io).toNanoseconds()), - prompt[0..], - false, - &stdout.interface, - platform, - ); -} - -fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { - const now: std.Io.Timestamp = .now(io, .awake); - log.info("Loading model config", .{}); - defer log.info("Loaded model config [{D}]", .{ - stdx.fmt.fmtDuration(now.untilNow(io, .awake)), - }); - - const parsed_config = blk: { - const config_json_file = try dir.openFile(io, "config.json", .{}); - defer config_json_file.close(io); - var config_json_buffer: [256]u8 = undefined; - var config_reader = config_json_file.reader(io, &config_json_buffer); - var reader: std.json.Reader = .init(allocator, &config_reader.interface); - defer reader.deinit(); - break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - }; - errdefer parsed_config.deinit(); - - return parsed_config; -} - -fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { - // log.info("Loading tokenizer", .{}); - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Loading tokenizer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{D}]", .{ - stdx.fmt.fmtDuration(now.untilNow(io, .awake)), - }); - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, io, bytes); -} - -const LlamaParameters = struct { - prefill_tokens: zml.Tensor, - decode_tokens: zml.Tensor, - token_index: zml.Tensor, - kv_cache: llama.KvCache, - rng: zml.Tensor.Rng, - attention_metadata: zml.attention.attention.Metadata, - attention_parameters: zml.attention.attention.Parameters, -}; - -const CompileModelResult = struct { - prefill_exe: zml.Exe, - decode_exe: zml.Exe, -}; - -fn compileModel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: llama.LlamaLM, - parameters: LlamaParameters, - progress: *std.Progress.Node, -) !CompileModelResult { - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled model [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); - - // Compile the model twice, one for prefill, one for generation. - var prefill_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling prefill...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled prefill [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ - parameters_.prefill_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.attention_parameters, - }); - } - }.call, .{ allocator, io, platform, llama_model, parameters, progress }); - errdefer if (prefill_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - var decode_future = try io.concurrent(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { - progress_.increaseEstimatedTotalItems(1); - var node_ = progress_.start("Compiling decode...", 1); - defer node_.end(); - const now_: std.Io.Timestamp = .now(io_, .awake); - defer log.info("Compiled decode [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); - return platform_.compile(allocator_, io_, llama_model_, .forward, .{ - parameters_.decode_tokens, - parameters_.token_index, - parameters_.kv_cache, - parameters_.rng, - parameters_.attention_metadata, - parameters_.attention_parameters, - }); - } - }.call, .{ allocator, io, platform, llama_model, parameters, progress }); - errdefer if (decode_future.cancel(io)) |v| { - v.deinit(); - } else |_| {}; - - const prefill_exe = try prefill_future.await(io); - const decode_exe = try decode_future.await(io); - - return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; -} - -pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - if (skip_llama3_encoding) { - // Copy so the ownership is the same in both branches. - return try allocator.dupe(u32, try encoder.encode(prompt)); - } - - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; - const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; - const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = (try encoder.encode("\n"))[0]; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); - - try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline }); - - try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); - - return tokens.toOwnedSlice(allocator); -} - -pub fn generateText( - allocator: std.mem.Allocator, - io: std.Io, - llama_buffers: *zml.Bufferized(LlamaLM), - prefill_exe: zml.exe.Exe, - decode_exe: zml.exe.Exe, - kv_cache_buffers: *zml.Bufferized(llama.KvCache), - attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), - tokenizer: zml.tokenizer.Tokenizer, - config: LlamaLM.Config, - options: LlamaLM.Options, - seed: u128, - prompt: []const u8, - skip_llama3_encoding: bool, - writer: *std.Io.Writer, - platform: *const zml.Platform, -) !void { - const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); - defer allocator.free(prompt_tok); - - var tokenizer_decoder = try tokenizer.decoder(); - defer tokenizer_decoder.deinit(); - - const max_seq_len = options.max_seq_len; - - // init RNG and buffers - var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io); - defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); - defer generated_token_slice.free(allocator); - - { - var prefill_args = try prefill_exe.args(allocator); - defer prefill_args.deinit(allocator); - - var prefill_results = try prefill_exe.results(allocator); - defer prefill_results.deinit(allocator); - - // prepare device buffers for the prefill tokens and their positions - const prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{max_seq_len}, .u32)); - defer prefill_tokens_slice.free(allocator); - @memcpy(prefill_tokens_slice.items(u32)[0..prompt_tok.len], prompt_tok); - - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); - defer prefill_tokens_buffer.deinit(); - var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); - defer prefill_token_pos_buffer.deinit(); - - prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); - - prefill_exe.call(prefill_args, &prefill_results); - - prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); - try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); - generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; - - std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); - const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); - std.log.info("\n\ngenerated: {s}\n\n", .{out}); - } - - // Prepare for token-by-token generation, - - var decode_args = try decode_exe.args(allocator); - defer decode_args.deinit(allocator); - - var decode_results = try decode_exe.results(allocator); - defer decode_results.deinit(allocator); - - // start with the token generated based on the full prompt. - var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); - defer current_token_buffer.deinit(); - - const output_tokens_len = max_seq_len - prompt_tok.len - 1; - const now: std.Io.Timestamp = .now(io, .awake); - - // One token has alreadyh been generated by the prefill. - var num_tokens_generated: usize = 1; - - generation: for (0..output_tokens_len + 1) |i| { - // collect and print generated sequence - num_tokens_generated += 1; - const generated_token = generated_token_slice.items(u32)[0]; - if (try tokenizer_decoder.next(generated_token)) |chunk| { - try writer.writeAll(chunk); - try writer.flush(); - } - - // check for eos - if (i == output_tokens_len) break :generation; - switch (config.eos_token_id.value) { - .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, - .ints => |eos_list| { - for (eos_list) |eos| { - if (generated_token == @as(u32, @intCast(eos))) break :generation; - } - }, - } - - // current token pos needs to go into a zml.Buffer - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); - defer token_pos_buffer.deinit(); - - // call to generate the next token - decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); - - decode_exe.call(decode_args, &decode_results); - - decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); - - // extract the generated token from the buffer - try current_token_buffer.toSlice(io, generated_token_slice); - } - const duration = now.untilNow(io, .awake); - std.debug.print("\n", .{}); - log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ - num_tokens_generated, - stdx.fmt.fmtDuration(duration), - stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(duration, num_tokens_generated)), - }); -} diff --git a/examples/llama/test.zig b/examples/llama/test.zig deleted file mode 100644 index 6c4ebc84..00000000 --- a/examples/llama/test.zig +++ /dev/null @@ -1,119 +0,0 @@ -const std = @import("std"); - -const async = @import("async"); -const zml = @import("zml"); -const Tensor = zml.Tensor; -const stdx = zml.stdx; -const flags = stdx.flags; - -const llama_mod = @import("./llama.zig"); -const LlamaLM = llama_mod.LlamaLM; - -pub fn main() !void { - try async.AsyncThread.main(std.heap.c_allocator, asyncMain); -} - -pub fn asyncMain() !void { - const CliArgs = struct { - pub const help = - \\ test-implementation --weights=llama3.8B.safetensors --config=config.json --reference=activation.safetensors - ; - weights: []const u8, - config: []const u8, - reference: []const u8, - num_heads: ?i64 = null, - num_kv_heads: ?i64 = null, - rope_freq_base: ?i64 = null, - }; - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - // Select platform - const platform = context.autoPlatform(.{}); - - // Parse program args - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - const model_file = cli_args.weights; - - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - std.log.info("Model file: {s}", .{model_file}); - - // Read model shapes. - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer buffer_store.deinit(); - - // Create the model and configure it. - var llama = try zml.aio.populateModel(LlamaLM, model_arena, buffer_store); - const num_heads: i64 = cli_args.num_heads orelse buffer_store.metadata("num_heads", .int) orelse @panic("--num_heads is required for this model"); - const num_kv_heads: i64 = cli_args.num_kv_heads orelse buffer_store.metadata("num_kv_heads", .int) orelse num_heads; - - const config = blk: { - var config_json_file = try async.File.open(cli_args.config, .{ .mode = .read_only }); - defer config_json_file.close() catch unreachable; - var reader = std.json.reader(allocator, config_json_file.reader()); - defer reader.deinit(); - const config_obj = try std.json.parseFromTokenSourceLeaky(LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); - break :blk config_obj; - }; - std.log.info("Parsed llama config: {}", .{config}); - - const llama_config: LlamaLM.Config = .{ - .eos_token_id = config.eos_token_id, - .bos_token_id = config.bos_token_id, - .num_key_value_heads = @intCast(num_kv_heads), - .num_hidden_layers = @intCast(config.num_hidden_layers), - .num_attention_heads = @intCast(num_heads), - .max_position_embeddings = config.max_position_embeddings, - .rope_theta = config.rope_theta, - .rms_norm_eps = @floatCast(buffer_store.metadata("rms_norm_eps", .float) orelse 1e-5), - .hf_rope_impl = true, - }; - - const llama_options: LlamaLM.Options = .{ - .max_seq_len = 256, - .sampling_strategy = .{ - .topk = 1, - .temperature = 1.0, - }, - }; - - std.log.info("Parsed llama config: {}", .{llama_options}); - llama.init(llama_config, llama_options); - - // Load the weights. - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{ llama_config, llama_options }, buffer_store, model_arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - - // Load the activations. - var activation_buffer_store = try zml.aio.torch.open(allocator, cli_args.reference); - defer activation_buffer_store.deinit(); - - // Test implementation - try testImplementation(platform, llama, llama_weights, activation_buffer_store); -} - -fn testImplementation( - platform: zml.Platform, - llama: LlamaLM, - llama_weights: zml.Bufferized(LlamaLM), - buffer_store: zml.aio.BufferStore, -) !void { - try zml.testing.testLayer(platform, buffer_store, "embed_tokens", llama.model.embed_tokens, llama_weights.model.embed_tokens, 1e-3); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.v_proj", llama.model.layers[0].self_attn.v_proj, llama_weights.model.layers[0].self_attn.v_proj, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.q_proj", llama.model.layers[0].self_attn.q_proj, llama_weights.model.layers[0].self_attn.q_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.k_proj", llama.model.layers[0].self_attn.k_proj, llama_weights.model.layers[0].self_attn.k_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.o_proj", llama.model.layers[0].self_attn.o_proj, llama_weights.model.layers[0].self_attn.o_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.mlp", llama.model.layers[0].mlp, llama_weights.model.layers[0].mlp, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.input_layernorm", llama.model.layers[0].input_layernorm, llama_weights.model.layers[0].input_layernorm, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.post_attention_layernorm", llama.model.layers[0].post_attention_layernorm, llama_weights.model.layers[0].post_attention_layernorm, 1e-2); -} diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel new file mode 100644 index 00000000..6a501aea --- /dev/null +++ b/examples/llm/BUILD.bazel @@ -0,0 +1,117 @@ +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") + +zig_binary( + name = "llm", + srcs = glob(["**/*.zig"]), + main = "main.zig", + visibility = ["//visibility:public"], + deps = ["//zml", "@linenoise"], +) + +zig_binary( + name = "lfm2_tests", + srcs = glob(["**/*.zig"]), + main = "models/lfm2_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +zig_binary( + name = "llama_tests", + srcs = glob(["**/*.zig"]), + main = "models/llama_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":llm"], +) + +mtree_spec( + name = "mtree", + srcs = [":llm"], + tags = ["manual"] +) + +tar( + name = "archive", + srcs = [":llm"], + args = [ + "--options", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"] +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["./{}/llm".format(package_name())], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [":archive"], + tags = ["manual"] +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/llm:latest"], + tags = ["manual"], +) + +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + repository = "index.docker.io/zmlai/llm", + tags = ["manual"], +) + + +oci_image( + name = "image_debug_", + base = "@debian_12", + entrypoint = ["./examples/llm/llm"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [":archive"], + tags = ["manual"], +) + +platform_transition_filegroup( + name = "image_debug", + srcs = [":image_debug_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load_debug", + image = ":image_debug", + repo_tags = ["zmlai/llm:debug"], + tags = ["manual"], +) diff --git a/examples/llm/README.md b/examples/llm/README.md new file mode 100644 index 00000000..fcd061ba --- /dev/null +++ b/examples/llm/README.md @@ -0,0 +1,41 @@ +# LLM + +`//examples/llm` runs interactive or one-shot text generation from a model repository from HuggingFace. + +We support the following models, automatically detected from the `model_type` in the `config.json`: + +- Llama 3.1 +- Qwen 3.5 +- LFM 2.5 + +## Run + +To load a model from HuggingFace directly: + +```bash +# CPU +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# CUDA +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# ROCm +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +``` + +From a local directory: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=/var/models/meta-llama/Llama-3.1-8B-Instruct/ +``` + +For a single non-interactive prompt: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Write a haiku about Zig" +``` + +## Options + +- `--model=`: Required. Model repository to load. This can be a local path or a huggingface/S3 URI such as `hf://...` or `s3://...`. +- `--prompt=`: Optional. Runs a single prompt instead of opening the interactive chat loop. +- `--seqlen=`: Optional. Maximum sequence length. Defaults to `2048`. +- `--backend=`: Optional. Attention backend. `cute` uses CuTe flash attention on CUDA GPUs with compute capability 8.0 or newer. If omitted, the program auto-selects one for the current platform, including `cute` on supported CUDA GPUs. diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig new file mode 100644 index 00000000..c02efd56 --- /dev/null +++ b/examples/llm/chat.zig @@ -0,0 +1,176 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); + +const models = @import("models.zig"); + +const log = std.log.scoped(.llm); + +pub const prompt_prefix: [:0]const u8 = "> "; +pub const history_file_path: [:0]const u8 = ".llm_chat_history"; + +pub fn initHistory() void { + _ = c.linenoiseHistorySetMaxLen(1_000); + _ = c.linenoiseHistoryLoad(history_file_path.ptr); +} + +pub fn rememberPrompt(line: [*:0]const u8) void { + if (line[0] == 0) return; + if (c.linenoiseHistoryAdd(line) == 1) { + _ = c.linenoiseHistorySave(history_file_path.ptr); + } +} + +pub const Chat = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + session: models.Session, + tokens: std.ArrayList(u32), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *models.CompiledModel, + model_buffers: *models.Buffers, + ) !Chat { + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, compiled_model.seqlen); + errdefer tokens.deinit(allocator); + + const session = try compiled_model.newSession( + allocator, + io, + platform, + model_buffers, + tokenizer, + ); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .session = session, + .tokens = tokens, + }; + } + + pub fn deinit(self: *Chat) void { + self.tokens.deinit(self.allocator); + self.session.deinit(); + } + + pub const RunOnceOptions = struct { + profile: bool = false, + }; + + pub fn runOnce(self: *Chat, prompt: []const u8, opts: RunOnceOptions) !void { + const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); + defer self.allocator.free(prompt_tokens); + + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); + var profiler: ?zml.Platform.Profiler = null; + defer if (profiler) |*p| p.deinit(); + + if (opts.profile) { + var session_id_buf: [64]u8 = undefined; + const session_id = try std.fmt.bufPrint( + &session_id_buf, + "llm-runonce-{d}", + .{@as(u64, @intCast(std.Io.Clock.now(.real, self.io).toNanoseconds()))}, + ); + profiler = try self.platform.profiler(self.allocator, self.io, .{ + .session_id = session_id, + }); + try profiler.?.start(); + } + + try self.runPrefill(prompt_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); + _ = if (profiler) |*p| try p.stop() else null; + + try stdout.interface.flush(); + } + + pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { + var turn_tokens = try self.session.tokenizePrompt(self.allocator, initial_prompt); + defer self.allocator.free(turn_tokens); + + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); + + while (self.tokens.items.len <= self.session.maxTokens()) { + try self.runPrefill(turn_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); + + turn_tokens = if (self.tokens.items.len >= self.session.maxTokens()) b: { + self.allocator.free(turn_tokens); + break :b try self.allocator.alloc(u32, 0); + } else try self.readNextTurn(turn_tokens, &stdout.interface) orelse return; + if (self.tokens.items.len + turn_tokens.len >= self.session.maxTokens()) { + log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ + self.session.maxTokens() -| self.tokens.items.len, + turn_tokens.len, + std.math.ceilPowerOfTwo(u32, self.session.maxTokens() + 1) catch std.math.maxInt(u32), + }); + return; + } + } + } + + fn runPrefill(self: *Chat, prompt_tokens: []const u32, stdout: *std.Io.Writer) !void { + if (prompt_tokens.len + self.tokens.items.len > self.session.maxTokens()) { + return error.PromptTooLong; + } + try stdout.writeAll("\x1b[2mprefill...\x1b[0m"); + try stdout.flush(); + + try self.tokens.appendSlice(self.allocator, prompt_tokens); + try self.session.runPrefill(self.tokens.items); + + try stdout.writeAll("\r \r"); + try stdout.flush(); + } + + fn runDecodeTurn(self: *Chat, stdout: *std.Io.Writer) !void { + const decode_start: std.Io.Timestamp = .now(self.io, .awake); + const decode_pos_before = self.tokens.items.len; + try self.session.runDecode(&self.tokens, stdout); + const decode_duration = decode_start.untilNow(self.io, .awake); + const decode_tokens: u64 = self.tokens.items.len - decode_pos_before; + + try stdout.writeAll("\n\n"); + try stdout.print("\x1b[36mdecode\x1b[0m \x1b[2m{f} · {:.1}tok/s\x1b[0m\n\n\n", .{ + decode_duration, + tokensPerSecond(decode_duration, decode_tokens), + }); + + try stdout.flush(); + } + + fn readNextTurn(self: *Chat, previous_turn_tokens: []const u32, stdout: *std.Io.Writer) !?[]const u32 { + while (true) { + const line = c.linenoise(prompt_prefix.ptr) orelse { + try stdout.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.tokens.items.len, self.session.maxTokens() }); + try stdout.flush(); + return null; + }; + defer c.linenoiseFree(line); + + rememberPrompt(line); + const input = std.mem.sliceTo(line, 0); + if (input.len == 0) continue; + + self.allocator.free(previous_turn_tokens); + return try self.session.tokenizeTurn(self.allocator, input); + } + } +}; + +fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { + if (tokens == 0) return 0; + const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; + if (seconds <= 0) return 0; + return @as(f64, @floatFromInt(tokens)) / seconds; +} diff --git a/examples/llm/main.zig b/examples/llm/main.zig new file mode 100644 index 00000000..b24c022a --- /dev/null +++ b/examples/llm/main.zig @@ -0,0 +1,205 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); +const stdx = zml.stdx; + +const chat = @import("chat.zig"); +const models = @import("models.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const log = std.log.scoped(.llm); + +const Args = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 2048, + topk: u32 = 4, + backend: ?enum { stablehlo, cute, nki, metal } = null, + profile: bool = false, + + pub const help = + \\ Use llm --model= [options] + \\ + \\ Run text generation with a model selected from `model_type` in the `config.json`. + \\ + \\ Options: + \\ --model= Path to the model repository (required) + \\ --prompt= Prompt to use for generation (default: none) + \\ --seqlen= Sequence length (default: 2048) + \\ --topk= Top-k sampling cutoff (default: 4) + \\ --backend= Attention backend to use ([stablehlo, cute, nki, metal], default: auto-selection) + \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + // `bazel run` executes binaries from Bazel's runfiles tree by default. + // If available, switch back to the shell's original working directory. + if (init.environ_map.get("BUILD_WORKING_DIRECTORY")) |build_working_directory| { + var working_dir = try std.Io.Dir.openDirAbsolute(init.io, build_working_directory, .{}); + defer working_dir.close(init.io); + try std.process.setCurrentDir(init.io, working_dir); + } + + const args = stdx.flags.parse(init.minimal.args, Args); + + // + // Virtual File Systems + // + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer http_client.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("gs", gcs_vfs.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); + + // + // Platform and Backend Selection + // + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, + } else zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{backend}); + + // + // Model initialization + // + log.info("Resolving model repository..", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + + log.info("Initializing model..", .{}); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + // Defines how the model's tensors are sharded across the available devices. + const meshes: models.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + const generation: models.GenerationOptions = .{ + .sampling_strategy = .{ + .topk = args.topk, + }, + }; + + var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + defer model.deinit(allocator); + + // + // Load the model and compile it + // + var progress = std.Progress.start(io, .{ .root_name = args.model }); + errdefer progress.end(); + + var tokenizer = try loadTokenizer(allocator, io, repo, &progress); + defer tokenizer.deinit(); + + var compiled_model = try allocator.create(models.CompiledModel); + defer allocator.destroy(compiled_model); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); + defer compiled_model.deinit(); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); + defer model.unloadBuffers(&model_buffers, allocator); + + progress.end(); + + // + // We're ready to go! Let's start a chat... + // + try printZmlLogo(io, init.environ_map); + + const interactive = args.prompt == null; + const prompt = if (args.prompt) |prompt_arg| b: { + break :b try allocator.dupe(u8, prompt_arg); + } else b: { + chat.initHistory(); + const line = c.linenoise(chat.prompt_prefix) orelse return; + defer c.linenoiseFree(line); + chat.rememberPrompt(line); + break :b try allocator.dupe(u8, std.mem.sliceTo(line, 0)); + }; + defer allocator.free(prompt); + + var llm_chat = try chat.Chat.init( + allocator, + io, + platform, + tokenizer, + compiled_model, + &model_buffers, + ); + defer llm_chat.deinit(); + + if (interactive) { + try llm_chat.runInteractive(prompt); + } else { + try llm_chat.runOnce(prompt, .{ + .profile = args.profile, + }); + } +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, bytes); +} + +pub fn printZmlLogo(io: std.Io, environ_map: *const std.process.Environ.Map) !void { + const stdout = std.Io.File.stdout(); + const logo_options: zml.logo.Options = .{ + .color_support = zml.logo.detectColorSupport(io, stdout, environ_map), + }; + var writer = stdout.writerStreaming(io, &.{}); + try writer.interface.writeAll("\n\n"); + try zml.logo.writeZmlArt(&writer.interface, logo_options); + try writer.interface.writeAll("\n\n\n"); + try writer.interface.flush(); +} diff --git a/examples/llm/models.zig b/examples/llm/models.zig new file mode 100644 index 00000000..20aa60c7 --- /dev/null +++ b/examples/llm/models.zig @@ -0,0 +1,201 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("models/common.zig"); +pub const Meshes = common.Meshes; +pub const GenerationOptions = common.GenerationOptions; +pub const parseConfig = common.parseConfig; +pub const lfm2 = @import("models/lfm2.zig"); +pub const llama = @import("models/llama.zig"); +pub const qwen3_5 = @import("models/qwen3_5.zig"); +pub const qwen3_5_moe = @import("models/qwen3_5_moe.zig"); + +const log = std.log.scoped(.llm); + +pub const ModelType = enum { + lfm2, + llama, + qwen3_5, + qwen3_5_moe, +}; + +const RawConfig = struct { + model_type: []const u8, +}; + +pub const LoadedModel = union(ModelType) { + lfm2: lfm2.LoadedModel, + llama: llama.LoadedModel, + qwen3_5: qwen3_5.LoadedModel, + qwen3_5_moe: qwen3_5_moe.LoadedModel, + + pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: GenerationOptions, + ) !LoadedModel { + const model_type = try detectModelType(allocator, io, repo); + log.info("Detected model type: {}", .{model_type}); + + return switch (model_type) { + inline else => |t| @unionInit( + LoadedModel, + @tagName(t), + try .init(allocator, io, repo, store, generation), + ), + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*m| m.deinit(allocator), + } + } + + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + return switch (self.*) { + inline else => |*m, t| @unionInit( + Buffers, + @tagName(t), + try m.loadBuffers(allocator, io, platform, store, progress), + ), + }; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), + } + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !CompiledModel { + const inner: CompiledModel.Inner = switch (self.*) { + inline else => |*m, t| @unionInit( + CompiledModel.Inner, + @tagName(t), + try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), + ), + }; + return .{ + .inner = inner, + .seqlen = @intCast(seqlen), + }; + } +}; + +pub const CompiledModel = struct { + const Inner = union(ModelType) { + lfm2: lfm2.inference.CompiledModel, + llama: llama.inference.CompiledModel, + qwen3_5: qwen3_5.inference.CompiledModel, + qwen3_5_moe: qwen3_5_moe.inference.CompiledModel, + }; + + inner: Inner, + seqlen: u32, + + pub fn deinit(self: *CompiledModel) void { + switch (self.inner) { + inline else => |*b| b.deinit(), + } + } + + pub fn newSession( + self: *CompiledModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *Buffers, + tokenizer: zml.tokenizer.Tokenizer, + ) !Session { + return switch (self.inner) { + inline else => |*compiled, t| .{ + .inner = @unionInit( + Session.Inner, + @tagName(t), + try .init(allocator, io, platform, tokenizer, compiled, &@field(model_buffers, @tagName(t))), + ), + .seqlen = self.seqlen, + }, + }; + } +}; + +pub const Buffers = union(ModelType) { + lfm2: lfm2.Buffers, + llama: llama.Buffers, + qwen3_5: qwen3_5.Buffers, + qwen3_5_moe: qwen3_5_moe.Buffers, +}; + +pub const Session = struct { + const Inner = union(ModelType) { + lfm2: lfm2.Session, + llama: llama.Session, + qwen3_5: qwen3_5.Session, + qwen3_5_moe: qwen3_5_moe.Session, + }; + + inner: Inner, + seqlen: u32, + + pub fn deinit(self: *Session) void { + switch (self.inner) { + inline else => |*s| s.deinit(), + } + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + try switch (self.inner) { + inline else => |*s| s.runPrefill(all_tokens), + }; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), writer: *std.Io.Writer) !void { + try switch (self.inner) { + inline else => |*s| s.runDecode(all_tokens, writer), + }; + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizePrompt(allocator, prompt), + }; + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizeTurn(allocator, prompt), + }; + } + + pub fn maxTokens(self: *const Session) u32 { + return self.seqlen; + } +}; + +pub fn detectModelType(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !ModelType { + const parsed = try parseConfig(RawConfig, allocator, io, repo); + defer parsed.deinit(); + if (std.meta.stringToEnum(ModelType, parsed.value.model_type)) |model_type| return model_type; + return error.UnknownModelType; +} diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig new file mode 100644 index 00000000..8a5c785d --- /dev/null +++ b/examples/llm/models/common.zig @@ -0,0 +1,88 @@ +const std = @import("std"); + +const zml = @import("zml"); + +pub const SessionOptions = struct { + seqlen: u32, + backend: zml.attention.Backend, +}; + +pub const GenerationOptions = struct { + sampling_strategy: zml.nn.SamplingStrategy = .{}, +}; + +pub const Phase = enum { + prefill, + decode, + + pub fn isPrefill(self: Phase) bool { + return self == .prefill; + } + + pub fn label(self: Phase) []const u8 { + return @tagName(self); + } + + pub fn startMessage(self: Phase, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "Compiling prefill " ++ component ++ "...", + .decode => "Compiling decode " ++ component ++ "...", + }; + } + + pub fn programName(self: Phase, comptime model_name: []const u8, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "llm_" ++ model_name ++ "_prefill_" ++ component, + .decode => "llm_" ++ model_name ++ "_decode_" ++ component, + }; + } + + pub fn logCompileDone(self: Phase, logger: anytype, comptime component: []const u8, io: std.Io, from: std.Io.Timestamp) void { + logger.info("Compiled {s} " ++ component ++ " [{f}]", .{ self.label(), from.untilNow(io, .awake) }); + } +}; + +pub const Meshes = struct { + model: *const zml.Mesh, + experts: *const zml.Mesh, + + pub fn init(platform: *zml.Platform) !Meshes { + switch (platform.target) { + .tpu => { + const has_link_y = platform.physical_mesh.hasAxis(.link_y); + + var strategy_experts: zml.Sharding.Strategy = .parseBindings(.{ .experts = .link_x }); + var strategy_model: zml.Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + if (has_link_y) { + strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); + strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); + } + + return .{ + .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + }; + }, + .cuda, .rocm, .oneapi, .neuron, .metal, .cpu, .furiosa => return .{ + .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), + }, + } + } + + pub fn all(self: Meshes) [2]*const zml.Mesh { + return .{ self.model, self.experts }; + } +}; + +pub fn parseConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { + const file = try dir.openFile(io, "config.json", .{}); + defer file.close(io); + + var buffer: [256]u8 = undefined; + var file_reader = file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + + return try std.json.parseFromTokenSource(T, allocator, &reader, .{ .ignore_unknown_fields = true }); +} diff --git a/examples/llm/models/lfm2.zig b/examples/llm/models/lfm2.zig new file mode 100644 index 00000000..4b58c18a --- /dev/null +++ b/examples/llm/models/lfm2.zig @@ -0,0 +1,11 @@ +pub const inference = @import("lfm2/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("lfm2/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("lfm2/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig new file mode 100644 index 00000000..0e37a999 --- /dev/null +++ b/examples/llm/models/lfm2/inference.zig @@ -0,0 +1,321 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const Phase = common.Phase; +const model = @import("model.zig"); + +const log = std.log.scoped(.lfm); + +pub const CompilationParameters = struct { + hidden_dim: usize, + batch_dim: usize, + rng: zml.Tensor.Rng, + cache: model.Cache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + seqlen: u32, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { + stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); + const cache: model.Cache = .{ + .kv = .init(.init(.{ + .layer = mdl.num_attention_layers, + .batch = 1, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = config.hidden_size / config.num_attention_heads, + }, mdl.embed_tokens.weight.dtype()), meshes.model), + .conv = .init(.init(.{ + .layer = mdl.num_conv_layers, + .batch = 1, + .seq = config.conv_L_cache, + .d = config.hidden_size, + }, mdl.embed_tokens.weight.dtype())), + }; + + return .{ + .hidden_dim = config.hidden_size, + .batch_dim = 1, + .rng = .init(), + .cache = cache, + .attention_metadata = .init(.fromBackend(backend)), + .attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + tokens_pos_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + rng_buf: *zml.Bufferized(zml.Tensor.Rng), + cache_buffers: *zml.Bufferized(model.Cache), + attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + loaded_model: *const model.LoadedModel, + mdl: model.Model, + opts: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, mdl, opts, 1, .decode, progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = opts, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.TokenEmbedding.forward), + conv: zml.FnExe(model.DecoderLayer.forward), + self_attn: zml.FnExe(model.DecoderLayer.forward), + sample: zml.FnExe(model.LmHead.forward), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.conv.deinit(); + self.self_attn.deinit(); + self.sample.deinit(); + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}), + layers: []zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), + sample: zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), buffers.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.layers) |*layer, layer_buffers| { + const layer_exe: *const zml.FnExe(model.DecoderLayer.forward) = switch (layer_buffers.operator) { + .conv => &exe.conv, + .self_attn => &exe.self_attn, + }; + layer.* = try zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ + .lm_head = buffers.lm_head, + .embed_tokens = buffers.embed_tokens, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run( + runner: *KernelRunner, + args: Args, + conv_cache_index_buffer: *zml.Buffer, + kv_cache_index_buffer: *zml.Buffer, +) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers) |*layer| { + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens_position_offset = args.tokens_pos_buf.*, + .active_length = args.active_length_buf.*, + .cache = args.cache_buffers.*, + .conv_cache_index = conv_cache_index_buffer.*, + .kv_cache_index = kv_cache_index_buffer.*, + .attention_metadata = args.attention_metadata_buffers, + }, + .outputs = .{ + .hidden = &hidden_buffer, + .cache = args.cache_buffers, + .conv_cache_index = conv_cache_index_buffer, + .kv_cache_index = kv_cache_index_buffer, + }, + }, .{}); + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, + .rng = args.rng_buf.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .rng = args.rng_buf, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const embed = try compileEmbed(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); + errdefer embed.deinit(); + const conv_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .conv, phase, progress); + errdefer conv_layer.deinit(); + const attn_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .full_attention, phase, progress); + errdefer attn_layer.deinit(); + const sample = try compileSample(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .conv = conv_layer, .self_attn = attn_layer, .sample = sample }; +} + +fn compileEmbed( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + embed_tokens: model.TokenEmbedding, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.TokenEmbedding.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", "embed_tokens"), + }, .{.{ + .embedding = embed_tokens, + .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + }}); +} + +fn compileLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + comptime kind: model.OperatorKind, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.DecoderLayer.forward) { + const label = switch (kind) { + .conv => "conv layer", + .full_attention => "attn layer", + }; + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage(label), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, label, io, from); + + const layer = for (mdl.layers) |candidate| { + const candidate_kind: model.OperatorKind = switch (candidate.operator) { + .conv => .conv, + .self_attn => .full_attention, + }; + if (candidate_kind == kind) break candidate; + } else unreachable; + + return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), + }, .{.{ + .layer = layer, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens_position_offset = zml.Tensor.init(.{ .batch = opts.batch_dim }, .u32), + .active_length = .init(.{}, .u32), + .cache = opts.cache, + .conv_cache_index = zml.Tensor.init(.{}, .u32), + .kv_cache_index = zml.Tensor.init(.{}, .u32), + .attention_metadata = opts.attention_metadata, + .attention_parameters = opts.attention_parameters, + .conv_parameters = .{ .is_prefill = phase.isPrefill() }, + }}); +} + +fn compileSample( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.LmHead.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", "lm_head"), + }, .{.{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.embed_tokens, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = opts.rng, + }}); +} diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig new file mode 100644 index 00000000..9f7cb909 --- /dev/null +++ b/examples/llm/models/lfm2/model.zig @@ -0,0 +1,761 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.lfm); + +pub const Config = struct { + architectures: []const []const u8, + block_auto_adjust_ff_dim: bool, + block_dim: u32, + block_ff_dim: u32, + block_ffn_dim_multiplier: f32, + block_mlp_init_scale: f32, + block_multiple_of: u32, + block_norm_eps: f32, + block_out_init_scale: f32, + block_use_swiglu: bool, + block_use_xavier_init: bool, + bos_token_id: u32, + conv_L_cache: u32, + conv_bias: bool, + conv_dim: u32, + conv_use_xavier_init: bool, + dtype: []const u8, + eos_token_id: u32, + hidden_size: u32, + initializer_range: f32, + intermediate_size: u32, + layer_types: []const []const u8, + max_position_embeddings: u32, + model_type: []const u8, + norm_eps: f32, + num_attention_heads: u32, + num_heads: u32, + num_hidden_layers: u32, + num_key_value_heads: u32, + pad_token_id: u32, + rope_theta: f32, + tie_embedding: bool, + transformers_version: []const u8, + use_cache: bool, + use_pos_enc: bool, + vocab_size: u32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + return .{ + .inner = .init(allocator, store, parsed_config.value, generation), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + Model.unloadBuffers(buffers, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); + return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + embed_tokens: TokenEmbedding, + lm_head: LmHead, + layers: []DecoderLayer, + num_attention_layers: usize, + num_conv_layers: usize, + + pub fn init( + allocator: std.mem.Allocator, + root_store: zml.io.TensorStore.View, + config: Config, + generation: common.GenerationOptions, + ) Model { + const store = root_store.withPrefix("model"); + stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); + + const layers = allocator.alloc(DecoderLayer, config.num_hidden_layers) catch unreachable; + var num_attention_layers: usize = 0; + var num_conv_layers: usize = 0; + for (layers, 0..) |*layer, i| { + const layer_store = store.withPrefix("layers").withLayer(i); + const kind = DecoderLayer.parseOperatorKind(config.layer_types[i]); + switch (kind) { + .conv => num_conv_layers += 1, + .full_attention => num_attention_layers += 1, + } + layer.* = DecoderLayer.init(config, layer_store, kind); + } + + return .{ + .embed_tokens = .init(store.withPrefix("embed_tokens")), + .lm_head = LmHead.init(store, config, generation.sampling_strategy), + .layers = layers, + .num_attention_layers = num_attention_layers, + .num_conv_layers = num_conv_layers, + }; + } + + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 8, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + LmHead.unloadBuffers(&self.lm_head); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + pub fn forward( + self: Model, + tokens: zml.Tensor, + tokens_position_offset: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + cache_: Cache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + conv_parameters: ConvParameters, + ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { + stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); + + const embeds = TokenEmbedding.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; + + var hidden = embeds; + var cache = cache_; + // Incremented by the individual layers. + var conv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); + var kv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); + for (self.layers) |layer| { + const result = DecoderLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .tokens_position_offset = tokens_position_offset, + .active_length = active_length, + .cache = cache, + .conv_cache_index = conv_cache_index, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .conv_parameters = conv_parameters, + }); + hidden = result.hidden; + cache = result.cache; + conv_cache_index = result.conv_cache_index; + kv_cache_index = result.kv_cache_index; + } + const result = LmHead.forward(.{ + .lm_head = self.lm_head, + .embed_tokens = self.embed_tokens, + .hidden = hidden, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + }); + return .{ result.tokens, cache.reuseBuffer(cache_), result.rng }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } +}; + +pub const TokenEmbedding = struct { + weight: zml.Tensor, + + pub const Input = struct { + embedding: TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; + } + + pub fn forward(input: Input) Output { + stdx.debug.assert(input.tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{input.tokens}); + stdx.debug.assert(input.embedding.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{input.embedding.weight}); + return .{ .hidden = input.embedding.weight.gather(.{ .voc = input.tokens }, .{}) }; + } + + pub fn unembed(self: TokenEmbedding, embeds: zml.Tensor) zml.Tensor { + stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); + return self.weight.dot(embeds, .d); + } +}; + +pub const LmHead = struct { + embedding_norm: RmsNorm, + sampling_strategy: zml.nn.SamplingStrategy, + + pub const Input = struct { + lm_head: LmHead, + embed_tokens: TokenEmbedding, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub fn init(store: zml.io.TensorStore.View, config: Config, sampling_strategy: zml.nn.SamplingStrategy) LmHead { + return .{ + .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), + .sampling_strategy = sampling_strategy, + }; + } + + pub fn forward(input: Input) Output { + const self = input.lm_head; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = input.hidden.slice(.seq, .dyn(last_index, 1)); + const logits = input.embed_tokens.unembed(self.embedding_norm.forward(hidden)); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, input.rng); + const tokens = new_tokens.convert(.u32); + return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { + RmsNorm.unloadBuffers(&self.embedding_norm); + } +}; + +pub const OperatorKind = enum { conv, full_attention }; +const Operator = union(enum) { conv: ShortConv, self_attn: Attention }; + +pub const DecoderLayer = struct { + operator: Operator, + operator_norm: RmsNorm, + ffn_norm: RmsNorm, + feed_forward: Mlp, + + pub const Input = struct { + layer: DecoderLayer, + hidden: zml.Tensor, + tokens_position_offset: zml.Tensor, + active_length: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + conv_parameters: ConvParameters, + }; + + pub const Output = struct { + hidden: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + }; + + pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { + return std.meta.stringToEnum(OperatorKind, layer_type) orelse { + stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); + unreachable; + }; + } + + pub fn init(config: Config, store: zml.io.TensorStore.View, kind: OperatorKind) DecoderLayer { + const operator_norm = RmsNorm.init(store.withPrefix("operator_norm"), config.norm_eps, .d); + const ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps, .d); + const feed_forward = Mlp.init(store.withPrefix("feed_forward")); + const operator: Operator = switch (kind) { + .conv => .{ .conv = ShortConv.init(config, store.withPrefix("conv")) }, + .full_attention => .{ .self_attn = Attention.init(config, store.withPrefix("self_attn")) }, + }; + + return .{ + .operator = operator, + .operator_norm = operator_norm, + .ffn_norm = ffn_norm, + .feed_forward = feed_forward, + }; + } + + pub fn forward(input: Input) Output { + const self = input.layer; + var cache = input.cache; + var conv_cache_index = input.conv_cache_index; + var kv_cache_index = input.kv_cache_index; + const residual = switch (self.operator) { + .conv => |operator| b: { + const residual, const updated_conv_cache = operator.forward( + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, + input.active_length, + cache.conv, + conv_cache_index, + input.conv_parameters, + ); + cache.conv = updated_conv_cache; + conv_cache_index = conv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + .self_attn => |operator| b: { + const residual, const updated_kv_cache = operator.forward( + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, + cache.kv, + kv_cache_index, + input.attention_metadata, + input.attention_parameters, + ); + cache.kv = updated_kv_cache; + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + }; + + const x = input.hidden.add(residual); + + return .{ + .hidden = x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input.hidden), + .cache = cache.reuseBuffer(input.cache), + .conv_cache_index = conv_cache_index.reuseBuffer(input.conv_cache_index), + .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + switch (self.operator) { + .conv => |*operator| ShortConv.unloadBuffers(operator), + .self_attn => |*operator| Attention.unloadBuffers(operator), + } + RmsNorm.unloadBuffers(&self.operator_norm); + RmsNorm.unloadBuffers(&self.ffn_norm); + Mlp.unloadBuffers(&self.feed_forward); + } +}; + +pub const ConvParameters = struct { is_prefill: bool }; + +pub const ShortConv = struct { + in_proj: Linear, + out_proj: Linear, + kernel: zml.Tensor, + config: Config, + + pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { + stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); + return .{ + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), + .config = config, + }; + } + + pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, active_length: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { + var cache = cache_; + const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); + + const B, const C, const x = BCx.chunkExact(.d, 3); + const Bx = B.mul(x); + + const conv_out = if (parameters.is_prefill) b: { + const active_length_i32 = active_length.convert(.i32); + const start = active_length_i32.sub(.scalar(self.config.conv_L_cache, .i32)).maximum(.scalar(0, .i32)); + const cache_seq_indices = lkp: { + const n = active_length_i32.sub(start); + const left_pad = zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); + break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); + }; + const scatter_data = Bx.slice(.seq, .dyn(start, self.config.conv_L_cache)); + cache.state = cache.state.scatterSlices(.{ .layer = cache_index, .seq = cache_seq_indices }, scatter_data, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); + const padding = self.config.conv_L_cache - 1; + const conv_out_ = zml.Tensor.conv1d(Bx, self.kernel, .{ + .padding = &.{ padding, 0 }, + .input_batch_dimension = Bx.axis(.batch), + .input_feature_dimension = B.axis(.d), + .input_spatial_dimensions = Bx.axis(.seq), + .kernel_output_feature_dimension = self.kernel.axis(.out), + .kernel_input_feature_dimension = self.kernel.axis(.in), + .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), + .output_batch_dimension = Bx.axis(.batch), + .output_feature_dimension = Bx.axis(.d), + .output_spatial_dimensions = Bx.axis(.seq), + .feature_group_count = Bx.dim(.d), + }); + break :b conv_out_.slice(.seq, .{ .end = Bx.dim(.seq) }); + } else b: { + cache.state = cache.state.rollRight1d(.seq, -1).scatterSlices(.{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, 0), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, Bx, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); + const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); + const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); + break :b cache.state.slice(.layer, .dynSingle(cache_index)).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); + }; + + const y = C.mul(conv_out); + + const output = self.out_proj.forward(y); + return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { + Linear.unloadBuffers(&self.in_proj); + Linear.unloadBuffers(&self.out_proj); + self.kernel.deinit(); + } +}; + +pub const Attention = struct { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + out_proj: Linear, + q_layernorm: RmsNorm, + k_layernorm: RmsNorm, + head_dim: usize, + num_key_value_groups: usize, + rope_opts: zml.nn.RopeOpts, + + pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { + const head_dim = config.hidden_size / config.num_attention_heads; + const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; + return .{ + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), + .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), + .head_dim = head_dim, + .num_key_value_groups = num_key_value_groups, + .rope_opts = .{ .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, + }; + } + + pub fn forward( + self: Attention, + x: zml.Tensor, + tokens_position_offset: zml.Tensor, + kv_cache: KvCache, + cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + + q = self.q_layernorm.forward(q); + k = self.k_layernorm.forward(k); + + const token_positions = b: { + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); + break :b zml.Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); + }; + + q = zml.nn.rope(q, token_positions, self.rope_opts); + k = zml.nn.rope(k, token_positions, self.rope_opts); + + q = q.rename(.{ .seq = .q }); + k = k.rename(.{ .seq = .k }); + v = v.rename(.{ .seq = .k }); + + const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); + k = new_kv_cache.keys(cache_index); + v = new_kv_cache.values(cache_index); + + stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const mesh = zml.Compiler.current().mesh(.model); + const partitioning: zml.attention.Partitioning = .head(.model); + const attn = switch (attention_parameters) { + .stablehlo => zml.attention.attention( + q, + k, + v, + tokens_position_offset, + attention_metadata, + attention_parameters, + mesh, + partitioning, + ).merge(.{ .d = .{ .h, .hd } }), + else => zml.attention.attention( + q.squeeze(.batch), + k.squeeze(.batch), + v.squeeze(.batch), + tokens_position_offset.squeeze(.batch), + attention_metadata, + attention_parameters, + mesh, + partitioning, + ).merge(.{ .d = .{ .h, .hd } }) + .rename(.{ .q = .seq }) + .insertAxes(.seq, .{.batch}), + }; + + return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { + Linear.unloadBuffers(&self.q_proj); + Linear.unloadBuffers(&self.k_proj); + Linear.unloadBuffers(&self.v_proj); + Linear.unloadBuffers(&self.out_proj); + RmsNorm.unloadBuffers(&self.q_layernorm); + RmsNorm.unloadBuffers(&self.k_layernorm); + } +}; + +pub const Cache = struct { + conv: ConvCache, + kv: KvCache, + + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { + ConvCache.unloadBuffers(&self.conv); + KvCache.unloadBuffers(&self.kv); + } + + pub fn reuseBuffer(self: Cache, other: Cache) Cache { + return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; + } +}; + +pub const ConvCache = struct { + state: zml.Tensor, + + pub fn init(shape: zml.Shape) ConvCache { + return .{ .state = .fromShape(shape) }; + } + + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { + const sh = self.state.shape(); + const host = try allocator.alloc(u8, sh.byteSize()); + defer allocator.free(host); + @memset(host, 0); + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { + self.state.deinit(); + } + + pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { + return .{ .state = self.state.reuseBuffer(other.state) }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); + return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; + } + + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; + } + + pub fn keys(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.k.slice(.layer, .dynSingle(cache_index)); + } + + pub fn values(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.v.slice(.layer, .dynSingle(cache_index)); + } + + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_position: zml.Tensor, cache_index: zml.Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + const layer = cache_index.broad(token_position.shape()); + return .{ + .k = self.k.scatterSlices(.{ .layer = layer, .k = token_position }, new_k.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), + .v = self.v.scatterSlices(.{ .layer = layer, .k = token_position }, new_v.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), + }; + } +}; + +pub const Linear = struct { + weight: zml.Tensor, + bias: ?zml.Tensor = null, + in_tag: zml.Shape.Tag, + out_tag: zml.Shape.Tag, + + pub fn init(weight: zml.Tensor, bias: ?zml.Tensor, tag: anytype) Linear { + stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); + const axis = weight.shape().axis(tag); + const out_tag = weight.shape().tag(1 - axis); + return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; + } + + pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { + linear.weight.deinit(); + if (linear.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { + var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); + return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; + } +}; + +const Mlp = struct { + w1: Linear, + w2: Linear, + w3: Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + Linear.unloadBuffers(&self.w1); + Linear.unloadBuffers(&self.w2); + Linear.unloadBuffers(&self.w3); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32, + tag: zml.Shape.Tag, + + pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { + return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); + return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); + } +}; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig new file mode 100644 index 00000000..ecc5abf8 --- /dev/null +++ b/examples/llm/models/lfm2/session.zig @@ -0,0 +1,227 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + config: *const model.Config, + seqlen: u32, + cache_buffers: zml.Bufferized(model.Cache), + rng_buf: zml.Bufferized(zml.Tensor.Rng), + generated_token_slice: zml.Slice, + think_start: ?u32, + think_end: ?u32, + tokenizer: zml.tokenizer.Tokenizer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); + errdefer model.Cache.unloadBuffers(&cache_buffers); + var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .tokenizer = tokenizer, + .config = &compiled_model.loaded_model.parsed_config.value, + .seqlen = compiled_model.params.seqlen, + .cache_buffers = cache_buffers, + .rng_buf = rng_buf, + .generated_token_slice = generated_token_slice, + .think_start = tokenizer.tokenId("") orelse unreachable, + .think_end = tokenizer.tokenId("") orelse unreachable, + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.Cache.unloadBuffers(&self.cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buf); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + return tokens.toOwnedSlice(allocator); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + const tokens = tokens_slice.items(u32); + @memset(tokens, self.config.pad_token_id); + @memcpy(tokens[0..all_tokens.len], all_tokens); + + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); + defer tokens_buf.deinit(); + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer tokens_pos_buf.deinit(); + + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), + } }, + }; + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + var next_token_buf: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &tokens_buf, + .next_token_buf = &next_token_buf, + .tokens_pos_buf = &tokens_pos_buf, + .active_length_buf = &active_length_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = attention_metadata_buffers, + }, &conv_cache_index_buffer, &kv_cache_index_buffer); + + defer next_token_buf.deinit(); + try next_token_buf.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{1})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); + + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + const params = self.compiled_model.params; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + + if (token_id == self.config.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer token_pos_buffer.deinit(); + + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .tokens_pos_buf = &token_pos_buffer, + .active_length_buf = &active_length_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = attention_metadata_buffers, + }, &conv_cache_index_buffer, &kv_cache_index_buffer); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; diff --git a/examples/lfm/test_runner.zig b/examples/llm/models/lfm2_tests.zig similarity index 58% rename from examples/lfm/test_runner.zig rename to examples/llm/models/lfm2_tests.zig index 0df68635..19b28df6 100644 --- a/examples/lfm/test_runner.zig +++ b/examples/llm/models/lfm2_tests.zig @@ -1,30 +1,109 @@ const std = @import("std"); const zml = @import("zml"); -const stdx = zml.stdx; -const attention = zml.attention.attention; -const model = @import("model.zig"); +const common = @import("common.zig"); +const lfm2 = @import("lfm2.zig"); +const model = @import("lfm2/model.zig"); -pub fn run(allocator: std.mem.Allocator, io: std.Io, platform: *zml.Platform, config: model.Config, mdl: model.Model, model_buffers: *zml.Bufferized(model.Model), store: *zml.io.TensorStore, attention_metadata: attention.Metadata, attention_parameters: attention.Parameters) !void { - _ = store; - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, "/Users/brabier/github/huggingface/transformers/lfm_activations.safetensors"); +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + backend: ?enum { stablehlo, cute, nki, metal } = null, + + pub const help = + \\Use lfm2_tests --model= --activations= [options] + \\ + \\ Validate the LFM2 implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ --backend= Attention backend to use + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); + + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, + } else zml.attention.Backend.auto(platform); + const params = lfm2.CompilationParameters.init( + repo_model.inner, + repo_model.parsed_config.value, + repo_model.parsed_config.value.max_position_embeddings, + backend, + meshes, + ); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); +} + +pub fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + config: lfm2.Config, + mdl: lfm2.Model, + model_buffers: *lfm2.Buffers, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + meshes: common.Meshes, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); defer registry.deinit(); - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); defer activation_store.deinit(); - var ctx = TestContext{ .allocator = allocator, .io = io, .platform = platform, .activations_store = &activation_store, .attention_metadata = attention_metadata, .attention_parameters = attention_parameters }; + var ctx = TestContext{ + .allocator = allocator, + .io = io, + .platform = platform, + .activations_store = &activation_store, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .mesh = platform.replicated_mesh, + }; try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); var num_attn_layers: usize = 0; var num_conv_layers: usize = 0; for (0..config.num_hidden_layers) |i| { - const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse { - stdx.debug.assert(false, "Unsupported layer type {s}", .{config.layer_types[i]}); - unreachable; - }; + const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse unreachable; const layer = mdl.layers[i]; const layer_buffers = model_buffers.layers[i]; switch (kind) { @@ -59,18 +138,11 @@ const TestContext = struct { io: std.Io, platform: *zml.Platform, activations_store: *zml.io.TensorStore, - attention_metadata: attention.Metadata, - attention_parameters: attention.Parameters, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + mesh: *const zml.Mesh, - fn testLayerPrint( - self: *TestContext, - comptime name_fmt: []const u8, - name_args: anytype, - tagz: anytype, - layer: anytype, - layer_buffers: anytype, - opts: zml.testing.CompareOpts, - ) !void { + fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); defer self.allocator.free(name); try self.testLayer(name, tagz, layer, layer_buffers, opts); @@ -81,14 +153,19 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }); + const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) + else + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); var args = try exe.args(self.allocator); @@ -98,9 +175,10 @@ const TestContext = struct { var res = try exe.results(self.allocator); defer res.deinit(self.allocator); - exe.call(args, &res); + exe.call(self.io, args, &res, .{}); - const out_result = res.get(zml.Buffer); + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); @@ -113,50 +191,54 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); defer self.allocator.free(cache_key); - const cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); + defer cache_buffer.deinit(); const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); - const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); - var args = try exe.args(self.allocator); - defer args.deinit(self.allocator); const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); defer actual_seq_len_buf.deinit(); + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); defer cache_index_buf.deinit(); - args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }); - var res = try exe.results(self.allocator); - defer res.deinit(self.allocator); + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }, + .{ &in_buffer, &cache_buffer }, + .{}, + ); - exe.call(args, &res); - - const out_result = res.get(zml.Buffer); - - try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); } @@ -167,53 +249,56 @@ const TestContext = struct { const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); defer self.allocator.free(in_key); - const in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); defer self.allocator.free(key_cache_key); - const key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); + defer key_cache_buffer.deinit(); const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); defer self.allocator.free(value_cache_key); - const value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); + defer value_cache_buffer.deinit(); const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); defer self.allocator.free(cache_pos_key); - const cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + defer cache_pos_buffer.deinit(); const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); defer self.allocator.free(out_key); - const out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }); + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); defer exe.deinit(); - var args = try exe.args(self.allocator); - defer args.deinit(self.allocator); const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); - defer attention.Metadata.deinitBuffer(&attention_metadata_buffers); - const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); defer cache_index_buf.deinit(); - args.set(.{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }); - var res = try exe.results(self.allocator); - defer res.deinit(self.allocator); + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }, + .{ &in_buffer, &key_cache_buffer, &value_cache_buffer }, + .{}, + ); - exe.call(args, &res); - - var out_result, var updated_kv = res.get(struct { zml.Buffer, zml.Bufferized(model.KvCache) }); - defer out_result.deinit(); - defer model.KvCache.unloadBuffers(&updated_kv); - - try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); std.log.info("Layer {s} passed!", .{name}); } }; @@ -227,6 +312,7 @@ fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml var io_buffer: [8 * 1024]u8 = undefined; var reader = try store.view().getReader(key, io, &io_buffer); defer reader.deinit(); + _ = try reader.interface.readSliceAll(host_bytes); return zml.Buffer.fromBytes(io, platform, shape, host_bytes); diff --git a/examples/llm/models/llama.zig b/examples/llm/models/llama.zig new file mode 100644 index 00000000..79f69e36 --- /dev/null +++ b/examples/llm/models/llama.zig @@ -0,0 +1,9 @@ +pub const inference = @import("llama/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const model = @import("llama/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("llama/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig new file mode 100644 index 00000000..51ea3f9c --- /dev/null +++ b/examples/llm/models/llama/inference.zig @@ -0,0 +1,209 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.llama); +const Phase = common.Phase; + +pub const CompilationParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + prefill_attention_parameters: zml.attention.Parameters, + decode_attention_parameters: zml.attention.Parameters, + seqlen: usize, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { + const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); + + return .{ + .prefill_tokens = .init(.{ .s = seqlen }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = mdl.model.layers.len, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = head_dim, + }, mdl.model.embed_tokens.weight.dtype()), meshes.model), + .rng = .init(), + .attention_metadata = .init(.fromBackend(backend)), + .prefill_attention_parameters = .init(.fromBackend(backend)), + .decode_attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + attention_metadata_buffers: *const zml.Bufferized(zml.attention.Metadata), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + llama_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress); + + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +const Forward = struct { + pub const Input = struct { + weights: model.Model, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + }; + + pub const Output = struct { + tokens: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + var hidden = model.EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = input.weights.model.embed_tokens }, + .tokens = input.tokens, + }).hidden; + var kv_cache = input.kv_cache; + for (input.weights.model.layers, 0..) |layer, index| { + const result = model.TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = input.token_index, + .kv_cache = kv_cache, + .kv_cache_index = .scalar(@as(u32, @intCast(index)), .u32), + .attention_metadata = input.attention_metadata, + .attention_parameters = input.attention_parameters, + }); + hidden = result.hidden; + kv_cache = result.kv_cache; + } + // LmHead owns final normalization; Llama.forward would apply it twice. + const result = model.LmHead.forward(.{ + .lm_head = .init(input.weights), + .hidden = hidden, + .active_length = input.active_length, + .rng = input.rng, + }); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) result.tokens.reuseBuffer(input.tokens) else result.tokens, + .kv_cache = kv_cache.reuseBuffer(input.kv_cache), + .rng = .{ ._state = result.rng._state.reuseBuffer(input.rng._state) }, + }; + } +}; + +pub const KernelExe = zml.FnExe(Forward.forward); + +pub const KernelRunner = struct { + forward: KernelExe.Runner(.{.weights}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + return .{ .forward = try .init(exe, allocator, .{ .weights = buffers.* }) }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.forward.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args) void { + runner.forward.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + .token_index = args.token_index_buf.*, + .active_length = args.active_length_buf.*, + .kv_cache = args.kv_cache_buffers.*, + .rng = args.rng_buffers.*, + .attention_metadata = args.attention_metadata_buffers.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .kv_cache = args.kv_cache_buffers, + .rng = args.rng_buffers, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: zml.attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("forward"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "forward", io, from); + + return KernelExe.compile(allocator, io, platform, .{ + .meshes = ¶meters.meshes.all(), + .program_name = phase.programName("llama", "forward"), + }, .{.{ + .weights = llama_model, + .tokens = zml.Tensor.init(.{ .s = seqlen }, .u32), + .token_index = parameters.token_index, + .active_length = .init(.{}, .u32), + .kv_cache = parameters.kv_cache, + .rng = parameters.rng, + .attention_metadata = parameters.attention_metadata, + .attention_parameters = attention_parameters, + }}); +} diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig new file mode 100644 index 00000000..8a6a09e9 --- /dev/null +++ b/examples/llm/models/llama/model.zig @@ -0,0 +1,651 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.llama); + +pub const Config = struct { + bos_token_id: u32, + eos_token_id: EosTokens, + head_dim: ?u32 = null, + hidden_size: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, + rope_theta: f32, + max_position_embeddings: u32, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + tie_word_embeddings: bool = false, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, + + pub const EosTokens = union(enum) { + int: u32, + ints: []u32, + + const Helpers = stdx.json.UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; +}; + +const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: u32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Options = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&buffers.model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); + + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +/// Llama architecture, using huggingface transformers naming. +/// Dimensions of activations: {.b, .s, .d} +pub const Model = struct { + lm_head: ?zml.nn.Linear, + model: Llama, + + gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + options: Options, + ) !Model { + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( + "weight", + .{ .dout, .d }, + .model, + .{ .dout = .model, .d = .replicated }, + )) |weight| + .init(weight, null, .d) + else + null; + + return .{ + .lm_head = lm_head, + .model = try .init(allocator, store.withPrefix("model"), config), + .gen_opts = options.sampling_strategy orelse .{}, + .config = config, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.model.deinit(allocator); + } + + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&self.model, allocator); + } + + fn lmHead(self: Model) LmHead { + return .init(self); + } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + + const out, const updated_kv_cache = self.model.forward( + tokens, + token_index, + kv_cache, + attention_metadata, + attention_parameters, + ); + + const sample = LmHead.forward(.{ + .lm_head = self.lmHead(), + .hidden = out, + .active_length = active_length, + .rng = rng, + }); + + return .{ sample.tokens, updated_kv_cache, sample.rng }; + } +}; + +const Llama = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor( + "embed_tokens.weight", + .{ .voc, .d }, + .model, + .{ .voc = .replicated, .d = .model }, + ) }, + .norm = .{ + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), + .eps = config.rms_norm_eps, + }, + .layers = layers, + }; + } + + pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + RmsNorm.unloadBuffers(&self.norm); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward( + self: Llama, + tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const embeds = self.embed_tokens.forward(tokens).withPartialTags(.{.d}); + var hidden = embeds; + var updated_kv_cache = kv_cache; + var kv_cache_index = zml.Tensor.scalar(0, .u32); + + for (self.layers) |layer| { + const result = TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = token_index, + .kv_cache = updated_kv_cache, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + }); + hidden = result.hidden; + updated_kv_cache = result.kv_cache; + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + } + + return .{ self.norm.forward(hidden), updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .replicate() }; + } +}; + +pub const LmHead = struct { + lm_head: ?zml.nn.Linear, + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + gen_opts: zml.nn.SamplingStrategy, + + pub fn init(mdl: Model) LmHead { + return .{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.model.embed_tokens, + .norm = mdl.model.norm, + .gen_opts = mdl.gen_opts, + }; + } + + pub const Input = struct { + lm_head: LmHead, + hidden: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + const self = input.lm_head; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d }).slice(.s, .dyn(last_index, 1))); + + var logits = blk: { + if (self.lm_head) |lm_head| { + break :blk lm_head.forward(hidden, hidden.dtype()).rename(.{ .dout = .d }); + } else { + break :blk self.embed_tokens.weight.withTags(.{ .voc, .d }).dot(hidden, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, input.rng); + return .{ .tokens = next_tokens.convert(.u32), .rng = new_rng }; + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: Mlp, + + pub fn init(store: zml.io.TensorStore.View, config: Config) !TransformerLayer { + return .{ + .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = try .init(store.withPrefix("self_attn"), config), + .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), + .mlp = .init(store.withPrefix("mlp")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + SelfAttn.unloadBuffers(&self.self_attn); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + Mlp.unloadBuffers(&self.mlp); + } + + pub const Input = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + }; + + pub const Output = struct { + hidden: zml.Tensor, + kv_cache: KvCache, + }; + + pub fn forward(input: Input) Output { + const self = input.layer; + const x0 = input.hidden; + // Self Attention + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + // Keep the residual stream replicated to avoid repeated gathers before q/k/v. + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const x0_normalized = self.input_layernorm.forward(x0_replicated); + const delta0, const updated_kv_cache = self.self_attn.forward( + x0_normalized, + input.token_index, + input.kv_cache, + input.kv_cache_index, + input.attention_metadata, + input.attention_parameters, + ); + + // Fully Connected + const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }) + .add(x1) + .withPartitioning(.model, .{ .d = .replicated }); + + return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ + .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), + .eps = eps, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +const Mlp = struct { + up_proj: zml.nn.Linear, // (dim -> hidden_dim) + gate_proj: zml.nn.Linear, // (dim -> hidden_dim) + down_proj: zml.nn.Linear, // (hidden_dim -> dim) + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output, output.dtype()); + } +}; + +const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: ?RmsNorm, + k_norm: ?RmsNorm, + + o_proj: zml.nn.Linear, + num_heads: i64 = undefined, + num_kv_heads: i64 = 0, + rope_opts: zml.nn.RopeOpts = undefined, + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + var rope_scaling = config.rope_scaling; + rope_scaling.setRopeTheta(config.rope_theta); + return .{ + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), + // TODO(Corentin): fix that + .q_norm = null, + .k_norm = null, + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .real_im_pass else .interleaved, + .scaling = rope_scaling, + }, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + /// Self zml.attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + + // Make hidden state replicated once and reuse it across q/k/v projections. + // This avoids paying gather-style collectives independently for each projection. + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.updateAt(k, v, token_index, kv_cache_index); + k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); + v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); + + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + attention_metadata, + attention_parameters, + zml.Compiler.current().mesh(.model), + .head(.model), + ); + + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + const delta = self.o_proj.forward(attn, attn.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }); + return .{ delta, new_kv_cache }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + + pub const Buffer = zml.Bufferized(KvCache); + + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); + + return .{ + .k = .fromShape(sharded_shape), + .v = .fromShape(sharded_shape), + }; + } + + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), + }; + } + + pub fn deinitBuffer(kv: *Buffer) void { + kv.k.deinit(); + kv.v.deinit(); + } + + pub fn keysAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.k.slice(.layer, .dynSingle(layer_index)); + } + + pub fn valuesAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.v.slice(.layer, .dynSingle(layer_index)); + } + + pub fn updateAt(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: zml.Tensor, layer_index: zml.Tensor) KvCache { + const k_shape = kv.k.shape().drop(.layer); + return .{ + .k = kv.k.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_v.convert(kv.v.dtype()).transpose(kv.v.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + }; + } + + pub fn reuseBuffer(kv: KvCache, other: KvCache) KvCache { + return .{ + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + }; + } +}; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig new file mode 100644 index 00000000..8a6532e6 --- /dev/null +++ b/examples/llm/models/llama/session.zig @@ -0,0 +1,222 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + kv_cache_buffers: zml.Bufferized(model.KvCache), + token_index_buffers: []zml.Buffer, + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + tokenizer: zml.tokenizer.Tokenizer, + config: *const model.Config, + seqlen: u32, + last_generated_token: u32 = 0, + conversation_id: u64, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); + errdefer allocator.free(token_index_buffers); + var initialized_token_index_buffers: usize = 0; + errdefer { + for (token_index_buffers[0..initialized_token_index_buffers]) |*token_index_buffer| { + token_index_buffer.deinit(); + } + } + for (token_index_buffers, 0..) |*token_index_buffer, i| { + token_index_buffer.* = try zml.Buffer.scalar(io, platform, i, .u32); + initialized_token_index_buffers = i + 1; + } + + const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .kv_cache_buffers = kv_cache_buffers, + .token_index_buffers = token_index_buffers, + .rng_buffers = rng_buffers, + .tokenizer = tokenizer, + .config = &compiled_model.loaded_model.parsed_config.value, + .seqlen = @intCast(compiled_model.params.seqlen), + .conversation_id = conversation_id, + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + for (self.token_index_buffers) |*token_index_buffer| { + token_index_buffer.deinit(); + } + self.allocator.free(self.token_index_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ self.config.bos_token_id, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); + defer active_length_buffer.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), + } }, + }; + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, + .token_index_buf = &self.token_index_buffers[0], + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, + }); + + defer next_token_buffer.deinit(); + self.last_generated_token = try next_token_buffer.getValue(u32, self.io); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); + defer self.allocator.free(decoder_out_buffer); + + var last_token_id: u32 = self.last_generated_token; + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); + defer current_token_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, 1, .u32); + defer active_length_buffer.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + generation: while (true) { + if (isEosToken(self.config, last_token_id)) break :generation; + + try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); + try stdout.flush(); + + try all_tokens.append(self.allocator, last_token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .token_index_buf = &self.token_index_buffers[all_tokens.items.len], + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, + }); + last_token_id = try current_token_buffer.getValue(u32, self.io); + } + + try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); + try stdout.flush(); + } +}; + +fn isEosToken(config: *const model.Config, token_id: u32) bool { + return switch (config.eos_token_id) { + .int => |eos| token_id == eos, + .ints => |eos_list| for (eos_list) |eos| { + if (token_id == eos) break true; + } else false, + }; +} diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig new file mode 100644 index 00000000..1b5894f1 --- /dev/null +++ b/examples/llm/models/llama_tests.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("common.zig"); +const llama = @import("llama.zig"); +const model = @import("llama/model.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + + pub const help = + \\Use llama_tests --model= --activations= + \\ + \\ Validate the LLaMA implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); +} + +fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + mdl: model.Model, + model_buffers: *model.Buffers, + meshes: []const *const zml.Mesh, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); + defer registry.deinit(); + + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); + defer activation_store.deinit(); + + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); + + if (mdl.model.layers.len == 0) return; + + const layer = mdl.model.layers[0]; + const layer_buffers = model_buffers.model.layers[0]; + + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); +} + +fn testLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + activation_store: zml.io.TensorStore.View, + name: []const u8, + layer: anytype, + layer_weights: zml.Bufferized(@TypeOf(layer)), + meshes: []const *const zml.Mesh, + opts: zml.testing.CompareOpts, +) !void { + const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); + defer allocator.free(in_key); + const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_shape); + + const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); + defer allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); + defer out_buffer_expected.deinit(); + + // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here + // is `fn (self, Tensor) Tensor`. + const Layer = @TypeOf(layer); + const Call = struct { + fn forward(l: Layer, x: zml.Tensor) zml.Tensor { + return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); + } + }; + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); + defer exe.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + args.set(.{ layer_weights, in_buffer }); + + var res = try exe.results(allocator); + defer res.deinit(allocator); + + exe.call(io, args, &res, .{}); + + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); + try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); +} + +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { + const shape = store.getShape(key) orelse return error.NotFound; + + const host_bytes = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(host_bytes); + + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.getReader(key, io, &io_buffer); + defer reader.deinit(); + + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); +} diff --git a/examples/llm/models/qwen3_5.zig b/examples/llm/models/qwen3_5.zig new file mode 100644 index 00000000..4147f695 --- /dev/null +++ b/examples/llm/models/qwen3_5.zig @@ -0,0 +1,11 @@ +pub const inference = @import("qwen3_5/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("qwen3_5/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("qwen3_5/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig new file mode 100644 index 00000000..50d3da30 --- /dev/null +++ b/examples/llm/models/qwen3_5/inference.zig @@ -0,0 +1,320 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5); +const Phase = common.Phase; + +pub const CompilationParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + seqlen: u32, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), + .rng = .init(), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + qwen_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), .decode, progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); + } +}; + +pub const LayerRunner = union(enum) { + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), + + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), + } + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), + layers: []LayerRunner, + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = .{ .embed_tokens = buffers.text_model.embed_tokens }, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .full_attention => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attention => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.Buffer) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers, layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = layer_index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); + }, + .linear_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = layer_index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .active_length = args.active_length_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); + }, + } + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, + .rng = args.rng_buffers.*, + .token_index = args.token_index_buf.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .rng = args.rng_buffers, + .token_index = args.token_index_buf, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const full_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + + const embed = try compileEmbed(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer embed.deinit(); + const full_attention = try compileFullAttention(allocator, io, platform, qwen_model, parameters, seqlen, full_index, phase, progress); + errdefer full_attention.deinit(); + const linear_attention = try compileLinearAttention(allocator, io, platform, qwen_model, parameters, seqlen, linear_index, phase, progress); + errdefer linear_attention.deinit(); + const sample = try compileSample(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .full_attention = full_attention, .linear_attention = linear_attention, .sample = sample }; +} + +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed tokens", io, from); + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}); +} + +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("full attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "full attention layer", io, from); + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .token_index = parameters.token_index, + .cache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + }}); +} + +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("linear attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "linear attention layer", io, from); + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .active_length = zml.Tensor.init(.{}, .u32), + .cache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + }}); +} + +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("sampler"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "sampler", io, from); + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + .sampler = mdl.sampler(), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = parameters.rng, + .token_index = parameters.token_index, + }}); +} + +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); +} + +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; + return null; +} diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig new file mode 100644 index 00000000..40174f5c --- /dev/null +++ b/examples/llm/models/qwen3_5/model.zig @@ -0,0 +1,1312 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.qwen3_5); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + zml.nn.Linear.unloadBuffers(&buffers.lm_head); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + lm_head: zml.nn.Linear, + + config: Config, + gen_options: GenOptions, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + const lm_head_prefix = b: { + if (store.hasKey("lm_head.weight")) break :b "lm_head"; + break :b "model.language_model.embed_tokens"; + }; + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), + .lm_head = .init( + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + null, + .d, + ), + .config = config, + .gen_options = gen_options, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, active_length, kv_cache); + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = text_model_output, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens, updated_kv_cache, result.rng }; + } + + pub fn sampler(self: Model) Sampler { + return .{ + .norm = self.text_model.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; + } +}; + +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, + .rng = new_rng, + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), + }; + } +}; + +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .replicate() }; + } +}; + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + ) !TextModel { + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + var hidden_states = EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = self.embed_tokens }, + .tokens = tokens, + }).hidden; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i)); + } + + return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub const TransformerLayer = struct { + const Attn = union(LayerType) { + linear_attention: GatedDeltaNet, + full_attention: SelfAttn, + }; + + input_layernorm: RmsNorm, + attn: Attn, + mlp: Mlp, + post_attention_layernorm: RmsNorm, + + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + active_length: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + const attn: Attn = switch (is_full_attention) { + true => .{ .full_attention = try .init(store.withPrefix("self_attn"), config) }, + false => .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, + }; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = attn, + .mlp = .init(store.withPrefix("mlp")), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Mlp.unloadBuffers(&self.mlp); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache.LayerView, + ) struct { zml.Tensor, KvCache } { + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .full_attention => |*self_attn| { + const cache = switch (kv_cache.cache) { + .self_attn => |cache| cache, + .linear_attn => unreachable, + }; + const result = self_attn.forward(normalized_x0, token_index, cache); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1].reuseBuffer(updated_kv_cache.self_attn); + }, + .linear_attention => |*linear_attn| { + const cache = switch (kv_cache.cache) { + .linear_attn => |cache| cache, + .self_attn => unreachable, + }; + const result = linear_attn.forward(normalized_x0, cache, active_length); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1].reuseBuffer(updated_kv_cache.gated_delta_net); + }, + } + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + } + + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const self_attn = switch (self.attn) { + .full_attention => |self_attn| self_attn, + .linear_attention => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const linear_attn = switch (self.attn) { + .linear_attention => |linear_attn| linear_attn, + .full_attention => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + gate_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), + .dout, + ), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); + const hidden = gate.silu().mul(up_projed); + + const output = self.down_proj.forward(hidden, hidden.dtype()); + return output; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), + .d, + ); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = b: { + if (self.num_kv_heads > 0) break :b self.num_kv_heads; + break :b self.num_heads; + }; + const k = self.k_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; + } + + fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var cache_tensor = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); + } + break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); + + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); + + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj + .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(tp, .{ .d = .replicated }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .real_im_pass, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved( + self: TextRotaryEmbedding, + position_ids: zml.Tensor, + dtype: zml.DataType, + ) struct { zml.Tensor, zml.Tensor } { + // To be used later in image extension. + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_z: zml.nn.Linear, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: u32, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + active_length: zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = b: { + if (initial_state) |state| { + break :b state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }); + } + + break :b zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + }; + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + active_length, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) + .rename(.{ .dout = .mix }) + .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = b: { + if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); + break :b projected_qkv; + }; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + + const z = self.in_proj_z.forward(x_in, x_in.dtype()) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); + + const query = mixed_qkv + .slice(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const key = mixed_qkv + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const value = mixed_qkv + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); + + const query_for_rule = b: { + if (self.qk_head_repetition == 1) break :b query; + break :b query.stutter1d(query.axis(.kh), self.qk_head_repetition); + }; + const key_for_rule = b: { + if (self.qk_head_repetition == 1) break :b key; + break :b key.stutter1d(key.axis(.kh), self.qk_head_repetition); + }; + + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( + query_for_rule, + key_for_rule, + value, + g, + beta, + b: { + if (use_cached_state) break :b cache.recurrentState(); + break :b null; + }, + active_length, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }); + + const output = self.out_proj + .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }); + const updated_cache = cache.update( + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(weight_f32.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub const Buffers = zml.Bufferized(SelfAttnCache); + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, tp: *const zml.Mesh) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + const kv_head_sharding = tp.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(tp, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(tp, .{ .h = .replicated }), + }; + return .{ + .k = .fromShape(sharded_kv_shape), + .v = .fromShape(sharded_kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(kv: *SelfAttnCache.Buffers) void { + kv.k.deinit(); + kv.v.deinit(); + kv.layer_index.deinit(); + } + + pub fn keys(kv: SelfAttnCache) zml.Tensor { + return kv.k.slice(.layer, .dynSingle(kv.layer_index)); + } + + pub fn values(kv: SelfAttnCache) zml.Tensor { + return kv.v.slice(.layer, .dynSingle(kv.layer_index)); + } + + pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = kv.k.shape().drop(.layer); + var layer = kv.layer_index; + if (token_index) |idx| { + layer = layer.broad(idx.shape()); + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; + } + + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; + } + pub fn atLayer(kv: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = kv.k, + .v = kv.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(kv: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + .layer_index = kv.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub const Buffers = zml.Bufferized(GatedDeltaNetCache); + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, tp: *const zml.Mesh) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = + 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(tp, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(tp, .{ .vh = .model }); + return .{ + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *GatedDeltaNetCache.Buffers) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = b: { + if (new_conv_state) |state| { + break :b self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state); + } + + break :b self.conv_state; + }; + + const recurrent_state = b: { + if (new_recurrent_state) |state| { + break :b self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state); + } + + break :b self.recurrent_state; + }; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + tp: *const zml.Mesh, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, tp), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, tp), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (denseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + return @intCast(std.mem.countScalar(LayerType, layer_types, layer_type)); + } + + fn denseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig new file mode 100644 index 00000000..5121fd30 --- /dev/null +++ b/examples/llm/models/qwen3_5/session.zig @@ -0,0 +1,219 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + layer_index_buffers: []zml.Buffer, + kv_cache_buffers: zml.Bufferized(model.KvCache), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + tokenizer: zml.tokenizer.Tokenizer, + generated_token_slice: zml.Slice, + seqlen: u32, + eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + const layer_index_buffers = try allocator.alloc(zml.Buffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*buffer| buffer.deinit(); + var full_attention_index: u32 = 0; + var linear_attention_index: u32 = 0; + for (layer_index_buffers, layer_types) |*buffer, layer_type| { + buffer.* = try switch (layer_type) { + .full_attention => b: { + defer full_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, full_attention_index, .u32); + }, + .linear_attention => b: { + defer linear_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, linear_attention_index, .u32); + }, + }; + initialized_layer_index_buffers += 1; + } + + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .layer_index_buffers = layer_index_buffers, + .kv_cache_buffers = kv_cache_buffers, + .rng_buffers = rng_buffers, + .tokenizer = tokenizer, + .generated_token_slice = generated_token_slice, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + for (self.layer_index_buffers) |*buffer| buffer.deinit(); + self.allocator.free(self.layer_index_buffers); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); + const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer prefill_token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, + .token_index_buf = &prefill_token_index_buffer, + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + }, self.layer_index_buffers); + + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .token_index_buf = &token_index_buffer, + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + }, self.layer_index_buffers); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; + +fn tokenizeChatPrompt( + allocator: std.mem.Allocator, + tokenizer: zml.tokenizer.Tokenizer, + prompt: []const u8, + special_tokens: model.Model.SpecialTokens, + is_first_turn: bool, +) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + const newline = tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); + if (!is_first_turn) { + try tokens.appendSlice(allocator, &.{ im_end, newline }); + } + + try tokens.append(allocator, im_start); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + + return tokens.toOwnedSlice(allocator); +} diff --git a/examples/llm/models/qwen3_5_moe.zig b/examples/llm/models/qwen3_5_moe.zig new file mode 100644 index 00000000..a97a3207 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe.zig @@ -0,0 +1,11 @@ +pub const inference = @import("qwen3_5_moe/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("qwen3_5_moe/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("qwen3_5_moe/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig new file mode 100644 index 00000000..896744b3 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -0,0 +1,354 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5_moe); + +pub const CompilationParameters = struct { + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + seqlen: u32, + meshes: common.Meshes, + xla_dump_to: ?[]const u8, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), + .rng = .init(), + .moe_backend = moe_backend, + .seqlen = seqlen, + .meshes = meshes, + .xla_dump_to = "/home/ubuntu/xla_dump", + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const LayerIndexBuffer = union(enum) { + self_attn: zml.Buffer, + linear_attn: zml.Buffer, +}; + +pub const RunArgs = struct { + io: std.Io, + tokens_buffer: *zml.Buffer, + next_token_buffer: *zml.Buffer, + token_index_buffer: *zml.Buffer, + active_length_buffer: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []const LayerIndexBuffer, +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + qwen_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, "prefill", progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, "decode", progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); + } +}; + +pub const LayerRunner = union(enum) { + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), + + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), + } + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), + layers: []LayerRunner, + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = buffers.text_model.embed_tokens, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .self_attn => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attn => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.text_model.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: RunArgs) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buffer.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers, args.layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .self_attn => |buffer| buffer, + .linear_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buffer.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); + }, + .linear_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .linear_attn => |buffer| buffer, + .self_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .active_length = args.active_length_buffer.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); + }, + } + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buffer.*, + .active_length = args.active_length_buffer.*, + .rng = args.rng_buffers.*, + .token_index = args.token_index_buffer.*, + }, + .outputs = .{ + .tokens = args.next_token_buffer, + .rng = args.rng_buffers, + .token_index = args.token_index_buffer, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + mdl: model.Model, + parameters: CompilationParameters, + seqlen: usize, + phase: []const u8, + progress: *std.Progress.Node, +) !KernelExe { + const full_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + + var embed_future = try io.concurrent(compileEmbed, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (embed_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, phase, progress }); + errdefer if (full_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, phase, progress }); + errdefer if (linear_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var sample_future = try io.concurrent(compileSample, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (sample_future.cancel(io)) |exe| exe.deinit() else |_| {}; + + const embed = try embed_future.await(io); + errdefer embed.deinit(); + const full_attention = try full_attention_future.await(io); + errdefer full_attention.deinit(); + const linear_attention = try linear_attention_future.await(io); + errdefer linear_attention.deinit(); + const sample = try sample_future.await(io); + errdefer sample.deinit(); + + return .{ + .embed = embed, + .full_attention = full_attention, + .linear_attention = linear_attention, + .sample = sample, + }; +} + +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { + return compileExe(allocator, io, platform, model.EmbedTokens.forward, .{.{ + .embedding = mdl.text_model.embed_tokens, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}, parameters, progress, phase, "embedding"); +} + +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .token_index = zml.Tensor.init(.{}, .u32), + .cache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_backend = parameters.moe_backend, + }}, parameters, progress, phase, "full-attention layer"); +} + +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .active_length = zml.Tensor.init(.{}, .u32), + .cache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_backend = parameters.moe_backend, + }}, parameters, progress, phase, "linear-attention layer"); +} + +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { + return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ + .sampler = mdl.text_model.sampler(), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = parameters.rng, + .token_index = zml.Tensor.init(.{}, .u32), + }}, parameters, progress, phase, "sampling"); +} + +fn compileExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + parameters: CompilationParameters, + progress: *std.Progress.Node, + phase: []const u8, + component: []const u8, +) !zml.FnExe(function) { + progress.increaseEstimatedTotalItems(1); + const label = try std.fmt.allocPrint(allocator, "Compiling {s} {s}...", .{ phase, component }); + defer allocator.free(label); + var node = progress.start(label, 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); + return zml.FnExe(function).compile(allocator, io, platform, .{ + .meshes = ¶meters.meshes.all(), + .xla_dump_to = parameters.xla_dump_to, + }, args); +} + +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); +} + +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; + return null; +} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig new file mode 100644 index 00000000..2a3b0ba8 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -0,0 +1,1423 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.qwen3_5_moe); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, + // MoE + num_experts: i64, + num_experts_per_tok: u32, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; +} + +fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var kv = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + kv = kv.stutter1d(kv.axis(.h), heads.factor); + } + break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; +} + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; + log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); + const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + + config: Config, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config, gen_options), + .config = config, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + } + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_backend); + return .{ new_tokens, updated_kv_cache, new_rng }; + } +}; + +//========================Text model======================== + +// Use this intermediate struct to compile sampleTokens without taking the whole TextModel as input +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, + .rng = new_rng, + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), + }; + } +}; + +pub const EmbedTokens = struct { + pub const Input = struct { + embedding: zml.nn.TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + return .{ .hidden = input.embedding.forward(input.tokens) }; + } +}; + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + gen_options: Model.GenOptions, + ) !TextModel { + const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + errdefer for (layers[0..i]) |previous_layer| previous_layer.deinit(allocator); + layer.* = try .init(allocator, store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), + .gen_options = gen_options, + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + for (self.layers) |layer| { + layer.deinit(allocator); + } + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer, allocator); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } + + pub fn sampler(self: TextModel) Sampler { + return .{ + .norm = self.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + config: Config, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + var hidden_states = EmbedTokens.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_backend); + } + + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = hidden_states, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens, updated_kv_cache.reuseBuffer(kv_cache), result.rng }; + } +}; + +pub const TransformerLayer = struct { + const Attn = union(enum) { + self_attn: SelfAttn, + linear_attn: GatedDeltaNet, + }; + + input_layernorm: RmsNorm, + attn: Attn, + moe: Moe, + post_attention_layernorm: RmsNorm, + + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + config: Config, + moe_backend: zml.moe.Backend, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + active_length: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + config: Config, + moe_backend: zml.moe.Backend, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = if (is_full_attention) + .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } + else + .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, + .moe = try .init(allocator, store.withPrefix("mlp"), config), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TransformerLayer, allocator: std.mem.Allocator) void { + self.moe.deinit(allocator); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Moe.unloadBuffers(&self.moe, allocator); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const self_attn = switch (self.attn) { + .self_attn => |self_attn| self_attn, + .linear_attn => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); + + return .{ + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const linear_attn = switch (self.attn) { + .linear_attn => |linear_attn| linear_attn, + .self_attn => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); + + return .{ + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache.LayerView, + config: Config, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache } { + _ = config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .self_attn => |*self_attn| { + const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1]; + }, + .linear_attn => |*linear_attn| { + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, active_length); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1]; + }, + } + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, moe_backend); + + return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + q_proj_scale: ?zml.Tensor, + k_proj: zml.nn.Linear, + k_proj_scale: ?zml.Tensor, + v_proj: zml.nn.Linear, + v_proj_scale: ?zml.Tensor, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + o_proj_scale: ?zml.Tensor, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), + .d, + ); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .q_proj_scale = null, + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj_scale = null, + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj_scale = null, + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), + .o_proj_scale = null, + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const k = self.k_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); + + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + up_proj_scale: ?zml.Tensor, + gate_proj: zml.nn.Linear, + gate_proj_scale: ?zml.Tensor, + down_proj: zml.nn.Linear, + down_proj_scale: ?zml.Tensor, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .up_proj_scale = null, + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .gate_proj_scale = null, + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), + .d, + ), + .down_proj_scale = null, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); + const hidden = gate.silu().mul(up_projed).rename(.{ .dout = .d }); + + const output = self.down_proj.forward(hidden, hidden.dtype()); + return output; + } +}; + +const Router = struct { + router: zml.nn.Linear, + num_experts_per_tok: u32, + + pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { + return .{ + .router = .init( + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), + .d, + ), + .num_experts_per_tok = num_experts_per_tok, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { + zml.Buffer.deinitAll(Router, self); + } + + pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const router_logits = self.router.forward(x, .f32); + const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); + const topk_ids = routing.indices.convert(.i32); + const router_scores = routing.values.softmax(.topk); + return .{ router_scores, topk_ids }; + } +}; + +pub const Moe = struct { + shared_expert: Mlp, + shared_expert_gate: zml.nn.Linear, + gate_up_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, + router: Router, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Moe { + _ = allocator; + const experts_store = store.withPrefix("experts"); + const gate_up_proj_tensor = experts_store.createTensor( + "gate_up_proj", + .{ .expert, .dout, .d }, + .experts, + .{ .expert = .experts, .dout = .replicated, .d = .replicated }, + ); + const down_proj_tensor = experts_store.createTensor( + "down_proj", + .{ .expert, .d, .dout }, + .experts, + .{ .expert = .experts, .d = .replicated, .dout = .replicated }, + ); + + return .{ + .shared_expert = Mlp.init(store.withPrefix("shared_expert")), + .shared_expert_gate = .init( + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), + .d, + ), + .gate_up_proj = .init(gate_up_proj_tensor, null, .d), + .down_proj = .init(down_proj_tensor, null, .dout), + .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok), + }; + } + + pub fn deinit(self: Moe, allocator: std.mem.Allocator) void { + _ = self; + _ = allocator; + } + + pub fn forward(self: Moe, x: zml.Tensor, moe_backend: zml.moe.Backend) zml.Tensor { + const routing_scores, const topk_ids = self.router.forward(x); + + const moe_output = zml.moe.forwardMoe( + x.merge(.{ .t = .{ .b, .s } }), + topk_ids.merge(.{ .t = .{ .b, .s } }), + routing_scores.merge(.{ .t = .{ .b, .s } }), + self.gate_up_proj, + self.down_proj, + moe_backend, + zml.Compiler.current().mesh(.experts), + .expert(.experts), + .{ + .activation = .{ .swiglu = .{} }, + .quantize_input = false, + .routing_weight_placement = .after_down, + }, + ).reshape(x.shape()); + + const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + }); + + return moe_output.add(shared); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { + _ = allocator; + zml.Buffer.deinitAll(Moe, self); + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .real_im_pass, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_qkv_scale: ?zml.Tensor, + in_proj_z: zml.nn.Linear, + in_proj_z_scale: ?zml.Tensor, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + out_proj_scale: ?zml.Tensor, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: u32, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_qkv_scale = null, + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_z_scale = null, + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .out_proj_scale = null, + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + active_length: zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = if (initial_state) |state| + state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) + else + zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + active_length, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) + .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = if (use_cached_state) + zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) + else + projected_qkv; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + + const z = self.in_proj_z.forward(x_in, x_in.dtype()) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + + const query = mixed_qkv + .slice(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); + const key = mixed_qkv + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); + const value = mixed_qkv + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); + + const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); + const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); + + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( + query_for_rule, + key_for_rule, + value, + g, + beta, + if (use_cached_state) cache.recurrentState() else null, + active_length, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) + .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); + const updated_cache = cache.update( + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(weight_f32.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + const kv_head_sharding = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), + }; + return .{ + .k = zml.Tensor.fromShape(sharded_kv_shape), + .v = zml.Tensor.fromShape(sharded_kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: SelfAttnCache) zml.Tensor { + return self.k.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn values(self: SelfAttnCache) zml.Tensor { + return self.v.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); + return .{ + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = if (new_conv_state) |state| + self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state) + else + self.conv_state; + + const recurrent_state = if (new_recurrent_state) |state| + self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state) + else + self.recurrent_state; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + model_meshe: *const zml.Mesh, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (getDenseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + var count: i64 = 0; + for (layer_types) |registered_layer_type| { + if (registered_layer_type == layer_type) count += 1; + } + return count; + } + + fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig new file mode 100644 index 00000000..6a819f26 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + kv_cache_buffers: zml.Bufferized(model.KvCache), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []inference.LayerIndexBuffer, + generated_token_slice: zml.Slice, + tokenizer: zml.tokenizer.Tokenizer, + seqlen: u32, + eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + const layer_index_buffers = try allocator.alloc(inference.LayerIndexBuffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + }; + + var self_attn_layer_index: u32 = 0; + var linear_attn_layer_index: u32 = 0; + for (layer_types, layer_index_buffers) |layer_type, *layer_index_buffer| { + layer_index_buffer.* = switch (layer_type) { + .full_attention => b: { + defer self_attn_layer_index += 1; + break :b .{ .self_attn = try .scalar(io, platform, self_attn_layer_index, .u32) }; + }, + .linear_attention => b: { + defer linear_attn_layer_index += 1; + break :b .{ .linear_attn = try .scalar(io, platform, linear_attn_layer_index, .u32) }; + }, + }; + initialized_layer_index_buffers += 1; + } + + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .kv_cache_buffers = kv_cache_buffers, + .rng_buffers = rng_buffers, + .layer_index_buffers = layer_index_buffers, + .generated_token_slice = generated_token_slice, + .tokenizer = tokenizer, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + for (self.layer_index_buffers) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + } + self.allocator.free(self.layer_index_buffers); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const tokens_slice = try zml.Slice.alloc(self.allocator, .init(.{ .b = 1, .s = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + @memset(tokens_slice.items(u32), 0); + @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); + defer tokens_buffer.deinit(); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buffer = &tokens_buffer, + .next_token_buffer = &next_token_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, + }); + + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + const out_tokens_buffer = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buffer = ¤t_token_buffer, + .next_token_buffer = ¤t_token_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, + }); + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; + +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len + 32); + if (!is_first_turn) { + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); + try tokens.appendSlice(allocator, newline); + } + + try tokens.append(allocator, im_start); + const user = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user); + try tokens.appendSlice(allocator, user); + const prompt_encoded = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_encoded); + try tokens.appendSlice(allocator, prompt_encoded); + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); + try tokens.appendSlice(allocator, newline); + try tokens.append(allocator, im_start); + const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant); + try tokens.appendSlice(allocator, assistant); + + return tokens.toOwnedSlice(allocator); +} diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index ad10f9f1..f0ddf811 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,5 +1,9 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:expand_template.bzl", "expand_template") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "mnist", @@ -19,7 +23,99 @@ zig_binary( ], ) -build_test( - name = "test", - targets = [":mnist"], +zig_binary( + name = "mnist_dp", + args = [ + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", + ], + data = [ + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], + main = "mnist_data_parallel.zig", + srcs = ["mnist.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//zml", + ], +) + +mtree_spec( + name = "mtree", + srcs = [":mnist"], + tags = ["manual"], +) + +tar( + name = "archive", + srcs = [":mnist"], + args = [ + "--options", + "zstd:compression-level=9", + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"], +) + +expand_template( + name = "entrypoint", + data = [ + ":mnist", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], + substitutions = { + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", + }, + template = [ + "./{}/mnist".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), + ], + tags = ["manual"], +) + +# The actual docker image, with entrypoint, created from tar archive +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ":entrypoint", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [":archive"], + tags = ["manual"], +) + +# We always want to create the image for Linux +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + target_platform = "@zml//platforms:linux_amd64", + tags = ["manual"], +) + +# Load will immediately load the image (eg: docker load) +oci_load( + name = "load", + image = ":image", + repo_tags = [ + "distroless/mnist:latest", + ], + tags = ["manual"], +) + +# Bazel target for pushing the Linux image to our docker registry +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + # override with -- --repository foo.bar/org/image + repository = "index.docker.io/steeve/mnist", + tags = ["manual"], ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 696de3af..f5842af8 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,12 +2,13 @@ const std = @import("std"); const log = std.log; const zml = @import("zml"); -const stdx = zml.stdx; pub const std_options: std.Options = .{ .log_level = .info, }; +pub const Img = [28 * 28]u8; + /// Model definition const Mnist = struct { fc1: Layer, @@ -19,8 +20,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createTensorWithTags("weight", .{ .d_out, .d }), - .bias = store.createTensorWithTags("bias", .{.d_out}), + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), }; } @@ -43,12 +44,16 @@ const Mnist = struct { platform: *const zml.Platform, store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, .{ - .store = store, - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer unloadBuffers(&buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -62,7 +67,10 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); @@ -83,22 +91,22 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); - defer store.deinit(); - const mnist_model: Mnist = .init(store.view()); - // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); + defer platform.deinit(allocator, io); + + // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled model [{D}]", .{stdx.fmt.fmtDuration(start.untilNow(io, .awake))}); - break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{}); }; defer exe.deinit(); @@ -106,10 +114,10 @@ pub fn main(init: std.process.Init) !void { var mnist_buffers = blk: { log.info("Transfering weights....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{D}]", .{ - stdx.fmt.fmtDuration(start.untilNow(io, .awake)), + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), }); - break :blk try mnist_model.load(allocator, io, platform, &store); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -130,16 +138,16 @@ pub fn main(init: std.process.Init) !void { // inference - can be looped const idx = rng.random().uintLessThan(u64, 10000); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); + var sample: Img align(16) = undefined; + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * @sizeOf(Img))); var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); - printDigit(sample); + printDigit(&sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(args, &results); + exe.call(io, args, &results, .{}); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); @@ -152,14 +160,14 @@ pub fn main(init: std.process.Init) !void { , .{digits[try result.getValue(u8, io)]}); } -fn printDigit(digit: [28 * 28]u8) void { +pub fn printDigit(digit_flat: *const Img) void { + const digit: *const [28][28]u8 = @ptrCast(digit_flat); var buffer: [28][30][2]u8 = undefined; for (0..28) |y| { buffer[y][0] = .{ '|', ' ' }; buffer[y][29] = .{ '|', '\n' }; for (1..29) |x| { - const idx = (y * 28) + (x - 1); - const val = digit[idx]; + const val = digit[y][x - 1]; buffer[y][x] = blk: { if (val > 240) break :blk .{ '*', '*' }; if (val > 225) break :blk .{ 'o', 'o' }; @@ -178,7 +186,7 @@ fn printDigit(digit: [28 * 28]u8) void { , .{std.mem.asBytes(&buffer)}); } -const digits = [_][]const u8{ +pub const digits = [_][]const u8{ \\ | ### | \\ | # # | \\ | # # | diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig new file mode 100644 index 00000000..b7805e49 --- /dev/null +++ b/examples/mnist/mnist_data_parallel.zig @@ -0,0 +1,206 @@ +//! Data-parallel version of mnist.zig, +//! Separated to kept `mnist.zig` minimal + +const std = @import("std"); + +const zml = @import("zml"); + +const base = @import("mnist.zig"); + +const log = std.log.scoped(.mnist); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +/// Model definition +const Mnist = struct { + fc1: Layer, + fc2: Layer, + + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + // Layer 1 is sharded following it's output axis + .fc1 = .{ + .weight = store.createTensor("fc1.weight", .{ .d_out, .d }, .dp_mp, .{ .d_out = .model }), + .bias = store.createTensor("fc1.bias", .{.d_out}, .dp_mp, .{ .d_out = .model }), + }, + // Layer 2 is sharded following it's input axis (and bias is fully replicated) + .fc2 = .{ + .weight = store.createTensor("fc2.weight", .{ .d_out, .d }, .dp_mp, .{ .d = .model }), + .bias = store.createReplicatedTensor("fc2.bias", .{.d_out}), + }, + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer zml.Buffer.deinitAll(Mnist, &buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + + /// just two linear layers + relu activation + pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { + var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + + const compiler = zml.Compiler.current(); + + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); + x = layer.forward(x); + } + + return x.argMax(.d).indices.convert(.u8) + .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) + .toMemory(.host_pinned); + } + + const Layer = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { + log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + const x = input.dot(self.weight, .d); + return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); + } + }; +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const allocator = init.gpa; + const io = init.io; + + // Parse program args + const process_args = try init.minimal.args.toSlice(arena.allocator()); + const model_path = process_args[1]; + const t10kfilename = process_args[2]; + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + + // Auto-select platform + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Decide how to partition the devices of our hardware. + // Here we use a mixed Data parallel / Model parallel + const dp_mp = try platform.registerMesh("dp_mp", try .init( + .{ .data, .model }, + &.{ .low_bandwidth, .high_bandwidth }, + &.{ 2, -1 }, + )); + log.info("{f}", .{dp_mp}); + + var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); + defer profiler.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + defer store.deinit(); + + const mnist_model: Mnist = .init(store.view()); + + // Compile model + const bs: u32 = 4; + const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + + var exe = exe: { + log.info("Compiling model....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); + break :exe try platform.compile( + allocator, + io, + mnist_model, + .forward, + .{input}, + .{ + .program_name = "mnist", + .meshes = &.{dp_mp}, + .xla_dump_to = "/tmp/zml/mnist_dp", + }, + ); + }; + defer exe.deinit(); + + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + // Load a random digit image from the dataset. + const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); + defer dataset.close(io); + + var rng: std.Random.DefaultPrng = blk: { + const now: std.Io.Timestamp = .now(io, .awake); + break :blk .init(@intCast(now.toMilliseconds())); + }; + + // inference - can be looped + var batch: []align(16) base.Img = try allocator.alignedAlloc(base.Img, .@"16", bs); + defer allocator.free(batch); + + for (0..bs) |i| { + const rand_id = rng.random().uintLessThan(u64, 10000); + _ = try dataset.readPositionalAll(io, @ptrCast(&batch[i]), 16 + (rand_id * @sizeOf(base.Img))); + } + + // This performs a host to host pinned memcpy. + // TODO: show how to allocate a Host pinned buffer and write into it. + var input_buffer: zml.Buffer = try .fromBytesOpts(io, platform, input.shape(), std.mem.sliceAsBytes(batch), .{ .memory = .host_pinned }); + defer input_buffer.deinit(); + + for (batch[0..]) |*sample| base.printDigit(sample); + + args.set(.{ mnist_buffers, input_buffer }); + + try profiler.start(); + exe.call(io, args, &results, .{}); + if (try profiler.stop()) |report| { + log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); + } + var recognized_digits_d: zml.Buffer = results.get(zml.Buffer); + defer recognized_digits_d.deinit(); + + const recognized_digits = try recognized_digits_d.getValue([bs]u8, io); + for (0..bs) |i| { + log.info( + \\✅ Shard {d} RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{ i, base.digits[recognized_digits[i]] }); + } +} diff --git a/examples/transfer-benchmark/BUILD.bazel b/examples/sharding/BUILD.bazel similarity index 67% rename from examples/transfer-benchmark/BUILD.bazel rename to examples/sharding/BUILD.bazel index 20082520..81f6de89 100644 --- a/examples/transfer-benchmark/BUILD.bazel +++ b/examples/sharding/BUILD.bazel @@ -2,13 +2,15 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_zig//zig:defs.bzl", "zig_binary") zig_binary( - name = "transfer-benchmark", + name = "sharding", + args = [], + data = [], main = "main.zig", - deps = ["//zml", "//bazel"], + deps = ["//zml"], visibility = ["//visibility:public"], ) build_test( name = "test", - targets = [":transfer-benchmark"], + targets = [":sharding"], ) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig new file mode 100644 index 00000000..34a78bfe --- /dev/null +++ b/examples/sharding/main.zig @@ -0,0 +1,263 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const log = std.log.scoped(.mesh); + +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/compiler", .level = .debug }, + }, +}; + +const CliMesh = enum { + auto, + mock, +}; + +const CliArgs = struct { + partitioner: ?zml.Sharding.Partitioner = null, + mesh: CliMesh = .auto, +}; + +const DemoModel = struct { + w: zml.Tensor, // {feature, hidden} + b: zml.Tensor, // {hidden} + + pub fn init(w: zml.Tensor, b: zml.Tensor) DemoModel { + return .{ .w = w, .b = b }; + } + + pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { + const compiler = zml.Compiler.current(); + + const x = input.convert(.f32); + var y = x.dot(self.w, .feature); + y = y.add(self.b.broad(y.shape())); + y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); + y.print("dense_out"); + + const gate = y.scale(0.01).sigmoid(); + + const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); + + return zml.ops.manualComputation( + (struct { + y: zml.Tensor, + gate: zml.Tensor, + + fn body(ctx: @This(), output_sharded: zml.Shape) zml.Tensor { + const local_result = ctx.y.relu().mul(ctx.gate); + return local_result.reshape(output_sharded); + } + }).body, + .{ .y = y, .gate = gate }, + y.shape(), + compiler.mesh(.dp_mp), + .{ + .in_specs = .{ .y = spec, .gate = spec }, + .out_specs = spec, + }, + ); + } +}; + +fn usage() void { + std.debug.print( + \\Usage: sharding [--partitioner=shardy|gspmd] [--mesh=auto|mock] + \\ + \\Simple sharding demo: + \\ - explicit partitioning on data/model axes + \\ - tensor print + \\ - manualComputation with multiple inputs and one output + \\ + \\Mesh modes: + \\ - auto: physical_mesh=auto, inferred cpu device_count=2 + \\ - mock: physical_mesh=mock topology, inferred cpu device_count=9 + \\ + , .{}); +} + +fn parseArgs(init: std.process.Init) !CliArgs { + var args: CliArgs = .{}; + var it = init.minimal.args.iterate(); + _ = it.next(); // program name + + while (it.next()) |arg| { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + usage(); + std.process.exit(0); + } else if (std.mem.startsWith(u8, arg, "--partitioner=")) { + const value = arg["--partitioner=".len..]; + if (std.mem.eql(u8, value, "shardy")) { + args.partitioner = .shardy; + } else if (std.mem.eql(u8, value, "gspmd")) { + args.partitioner = .gspmd; + } else { + std.debug.print("error: unknown partitioner '{s}'\n\n", .{value}); + usage(); + return error.InvalidPartitioner; + } + } else if (std.mem.startsWith(u8, arg, "--mesh=")) { + const value = arg["--mesh=".len..]; + if (std.mem.eql(u8, value, "auto")) { + args.mesh = .auto; + } else if (std.mem.eql(u8, value, "mock")) { + args.mesh = .mock; + } else { + std.debug.print("error: unknown mesh mode '{s}'\n\n", .{value}); + usage(); + return error.InvalidMeshMode; + } + } else { + std.debug.print("error: unknown argument '{s}'\n\n", .{arg}); + usage(); + return error.InvalidArgument; + } + } + return args; +} + +fn buildMockMesh( + allocator: std.mem.Allocator, + target: zml.Target, + devices: []const zml.platform.Device, +) !zml.Sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevicesForMockMesh; + const topology: zml.Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[3]), .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), .device(devices[0]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), .device(devices[7]), + }), + }), + }); + return zml.Sharding.PhysicalMesh.fromTree(allocator, target, topology); +} + +fn createSequenceBuffer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + shape: zml.Shape, + start: f32, +) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = start + @as(f32, @floatFromInt(i)); + } + + return zml.Buffer.fromSlice(io, platform, slice); +} + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + var args = try parseArgs(init); + + const create_options: zml.platform.CreateOptions = switch (args.mesh) { + .auto => .{}, + .mock => .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = buildMockMesh }, + }, + }; + + var platform: *zml.Platform = try .auto(allocator, io, create_options); + defer platform.deinit(allocator, io); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + var profiler_options: zml.Platform.ProfilerOptions = .defaults; + profiler_options.repository_path = "/tmp/xprof"; + profiler_options.session_id = "profiling"; + + var profiler = try platform.profiler(allocator, io, profiler_options); + defer profiler.deinit(); + + try profiler.start(); + defer { + _ = profiler.stop() catch unreachable; + } + + if (args.partitioner) |partitioner| { + log.info("Partitioner: {s}", .{@tagName(partitioner)}); + } else { + args.partitioner = .fromTarget(platform.target); + log.info("Partitioner: {s} (default)", .{@tagName(args.partitioner.?)}); + } + log.info("{f}", .{platform.physical_mesh}); + + const mesh: *const zml.Mesh = try platform.registerMesh( + "dp_mp", + .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + ); + + log.info("{f}", .{mesh.logical}); + log.info("{f}", .{mesh}); + + const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) + .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); + const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) + .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); + const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) + .withPartitioning(mesh, .{ .hidden = .model }); + + const input: zml.Tensor = zml.Tensor.fromShape(input_shape); + const w: zml.Tensor = zml.Tensor.fromShape(w_shape); + const b: zml.Tensor = zml.Tensor.fromShape(b_shape); + const model: DemoModel = .init(w, b); + + var exe = try platform.compile( + allocator, + io, + model, + .forward, + .{input}, + .{ + .partitioner = args.partitioner, + .meshes = &.{mesh}, + }, + ); + defer exe.deinit(); + + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); + defer w_buf.deinit(); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); + defer b_buf.deinit(); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); + defer input_buf.deinit(); + + log.info("input placement: {f}", .{input_buf}); + log.info("weight placement: {f}", .{w_buf}); + log.info("bias placement: {f}", .{b_buf}); + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + exe_args.set(.{ w_buf, b_buf, input_buf }); + + exe.call(io, exe_args, &exe_results, .{}); + var out = exe_results.get(zml.Buffer); + defer out.deinit(); + + const out_slice = try out.toSliceAlloc(allocator, io); + defer out_slice.free(allocator); + const out_items = out_slice.items(f32); + const preview_len = @min(out_items.len, @as(usize, 16)); + log.info("output preview: {any}", .{out_items[0..preview_len]}); +} diff --git a/examples/transfer-benchmark/main.zig b/examples/transfer-benchmark/main.zig deleted file mode 100644 index 52f5633b..00000000 --- a/examples/transfer-benchmark/main.zig +++ /dev/null @@ -1,190 +0,0 @@ -const std = @import("std"); -const log = std.log; - -const zml = @import("zml"); -const stdx = zml.stdx; - -const Args = struct { - pub const help = - \\ benchmark --model=./model - ; - model: []const u8, -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - - const io = vfs.io(); - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - const args = stdx.flags.parseProcessArgs(init.minimal, Args); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, args.model); - defer registry.deinit(); - - var store = zml.io.TensorStore.fromRegistry(allocator, ®istry); - defer store.deinit(); - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - defer progress.end(); - var transfer_progress = progress.start("Transferring tensors (MiB)...", @intCast(registry.totalBytes() / zml.MiB)); - defer transfer_progress.end(); - - log.info("⏱️ Running benchmark...", .{}); - - const now: std.Io.Timestamp = .now(io, .awake); - - var total_bytes: usize = 0; - const tensor_count = try transferAll( - allocator, - io, - platform, - .{ - .parallelism = 16, - .store = &store, - .dma_chunks = 4, - .dma_chunk_size = 32 * zml.MiB, - .progress = &transfer_progress, - .total_bytes = &total_bytes, - }, - ); - - const elapsed = now.untilNow(io, .awake); - const elapsed_ns = elapsed.toNanoseconds(); - const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; - - log.info("✅ Benchmark done!", .{}); - - const transfer_mib = @as(f64, @floatFromInt(total_bytes)) / @as(f64, zml.MiB); - const mib_per_s = transfer_mib / elapsed_s; - log.info("Transferred {d} tensors ({d} MiB) - Elapsed: {D} - {d:.3} MiB/s", .{ - tensor_count, - total_bytes >> 20, - stdx.fmt.fmtDuration(elapsed), - mib_per_s, - }); -} - -fn transferAll( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - opts: zml.io.LoadOpts, -) !usize { - const first_device = platform.devices[0]; // Temporary until sharding is re-exposed - const dma_allocator: zml.mem.DmaAllocator = .init(allocator, &first_device); - var buffer_pool: zml.mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); - defer buffer_pool.deinit(dma_allocator.allocator()); - - const keys = opts.store.registry.tensors.keys(); - const buffers = try allocator.alloc(zml.Buffer, keys.len); - const buffers_initialized = try allocator.alloc(bool, keys.len); - @memset(buffers_initialized, false); - defer { - for (buffers, buffers_initialized) |*buffer, is_initialized| { - if (is_initialized) buffer.deinit(); - } - allocator.free(buffers_initialized); - allocator.free(buffers); - } - - var ctx: TransferCtx = .{ - .dma_allocator = dma_allocator.allocator(), - .pinned_buffer_pool = &buffer_pool, - .io = io, - .memory = first_device.memory(.default), - .group = .init(opts.parallelism), - .progress = opts.progress, - .store = opts.store, - .buffers = buffers, - .buffers_initialized = buffers_initialized, - }; - - for (keys, 0..) |key, i| { - ctx.group.concurrent(io, transferOne, .{ i, key, &ctx }) catch unreachable; - } - - ctx.group.await(io) catch unreachable; - - if (opts.total_bytes) |total_bytes_ptr| { - total_bytes_ptr.* = ctx.total.load(.monotonic); - } - - return keys.len; -} - -const TransferCtx = struct { - dma_allocator: std.mem.Allocator, - pinned_buffer_pool: *zml.mem.DynamicBufferPool, - io: std.Io, - memory: *const zml.Memory, - group: stdx.Io.LimitedGroup, - total: std.atomic.Value(usize) = .init(0), - progress: ?*std.Progress.Node, - store: *const zml.io.TensorStore, - buffers: []zml.Buffer, - buffers_initialized: []bool, -}; - -fn transferOne(i: usize, key: []const u8, ctx: *TransferCtx) !void { - var reader = ctx.store.getReader(key, ctx.io, &.{}) catch unreachable; - defer reader.deinit(); - - var memory_writer = zml.io.MemoryWriter.init( - ctx.dma_allocator, - ctx.io, - ctx.memory, - ctx.pinned_buffer_pool, - reader.tensor.shape, - &ctx.buffers[i], - ) catch unreachable; - defer memory_writer.deinit(ctx.dma_allocator); - ctx.buffers_initialized[i] = true; - - const scale = zml.MiB; - const total = if (ctx.progress) |progress| blk: { - const child_total = @max(@as(u64, 1), reader.tensor.shape.byteSize() / scale); - var child = progress.start(reader.tensor.name, child_total); - defer child.end(); - - var progress_writer: zml.io.ProgressWriter = .init(memory_writer.interface(), &child, .{ .scale = scale }); - const written = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; - progress_writer.interface.flush() catch unreachable; - break :blk written; - } else blk: { - const written = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; - memory_writer.interface().flush() catch unreachable; - break :blk written; - }; - - const previous = ctx.total.fetchAdd(total, .monotonic); - if (ctx.progress) |progress| { - progress.setCompletedItems((previous + total) / scale); - } -} diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel index 1bf8ca7f..f53c70ed 100644 --- a/ffi/BUILD.bazel +++ b/ffi/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "cc", + tags = ["manual"], hdrs = [ "zig_allocator.h", "zig_slice.h", @@ -12,6 +13,7 @@ cc_library( zig_library( name = "zig", + tags = ["manual"], srcs = [ "zig_allocator.zig", "zig_slice.zig", diff --git a/ffi/zig_allocator.zig b/ffi/zig_allocator.zig index 35f5fc33..52c000bd 100644 --- a/ffi/zig_allocator.zig +++ b/ffi/zig_allocator.zig @@ -1,4 +1,5 @@ const std = @import("std"); + const c = @import("c"); pub const ZigAllocator = struct { @@ -12,7 +13,7 @@ pub const ZigAllocator = struct { pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) ?*anyopaque { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); - const ret = self.rawAlloc(elem * nelems, std.math.log2_int(usize, alignment), @returnAddress()) orelse return null; + const ret = self.rawAlloc(elem * nelems, .fromByteUnits(alignment), @returnAddress()) orelse return null; return @ptrCast(ret); } @@ -20,6 +21,6 @@ pub const ZigAllocator = struct { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); const memory: [*c]u8 = @ptrCast(ptr); const size = elem * nelems; - self.rawFree(memory[0..size], std.math.log2_int(usize, alignment), @returnAddress()); + self.rawFree(memory[0..size], .fromByteUnits(alignment), @returnAddress()); } }; diff --git a/mlir/.clang-format b/mlir/.clang-format new file mode 100644 index 00000000..9b3aa8b7 --- /dev/null +++ b/mlir/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: LLVM diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 87982a7e..8333c579 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -4,12 +4,9 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", hdrs = ["mlir.h"], - visibility = ["//mlir:__subpackages__"], + tags = ["manual"], deps = [ - "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPIMath", - "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", ], ) @@ -17,6 +14,7 @@ cc_library( zig_library( name = "mlir", main = "mlir.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":c", diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md new file mode 100644 index 00000000..7df47c5c --- /dev/null +++ b/mlir/dialects/README.md @@ -0,0 +1,269 @@ +# MLIR dialects + +One Bazel package per MLIR dialect that ZML emits. The rules come in three +tiers: + +- **Guarantees** hold for every package. Tests and the conformance checks + below enforce them, and callers rely on them. +- **Conventions** are the defaults. Follow them in new code; existing code + that differs is fine unless it confuses callers. +- **Exceptions** list the known, justified departures from the conventions. + +References: `cuda/`, `nvvm/` and `cute_nvgpu/` (generated from ODS, with +their own TableGen and C API), `gpu/`, `arith/` and `func/` (upstream +dialects, with a small C API for what upstream lacks). + +## Guarantees + +- **Registration.** The package exposes `registerDialect(registry)`, which + calls `registry.registerDialect("")` with a string literal, where + `` is the second argument of + `MLIR_DECLARE_CAPI_DIALECT_REGISTRATION`. Tests and consumers register + through `.registerDialect`, never through raw strings or + `DialectHandle.fromString`. A DSL exposes `registerDialects(registry)`, + which calls the `registerDialect` of each package it emits. +- **Operation list.** A package that builds operations has, in its + `test.zig`, a sorted `const operation_names: []const []const u8` naming + every operation its builders create. `test "every bound operation is + registered"` checks each name against the registry, and checks that + unregistered dialects are not allowed. A builder for an operation the pinned dialect does not register + does not exist; a DSL method builds it from the operations the Python DSL + lowers it to instead. +- **Bindings, not text.** A dialect package builds attributes and types + through C API constructors (`get`/`getChecked`), never by printing and + parsing text. A type or attribute the C API lacks gets a binding (a C + constructor and a Zig wrapper) before anything uses it. `mlir.Type.parse` + and `mlir.Attribute.parse` are for tests. Consumers use the bindings too. +- **Everything compiles.** `test.zig` runs `std.testing.refAllDecls` on the + package and on each of its wrapper types, so no declaration hides behind + lazy analysis. +- **Argument order.** A builder takes `ctx`, the required operands, the + result types (omitted when inferred), the required attributes, then the + optional arguments (an `opts` struct), then `location`, which is always + last. +- **Verification.** A builder verifies the operation it creates. One that + cannot (its region is filled later, or it only verifies inside a parent) + passes `.verify = false` and says so in its doc comment. +- **C API safety.** Every declaration is `MLIR_CAPI_EXPORTED`. Constructors + load the dialect, return a null handle for a null context, and verifying + constructors (`getChecked`) return a null handle for invalid input. No + `alwayslink` and no static initializers: the Zig side calls the + registration function by name, so the linker keeps what it uses. +- **Hygiene.** LLVM file banners and include-path guards (below), clean under + `mlir/.clang-format` and the Bazel toolchain's `zig fmt` (0.17), tests only + in `test.zig`, and the BUILD essentials: `capi` and the `zig_library` + tagged `manual`, an explicit `srcs` list in `td_files`, no `glob(`. + +## Conventions + +### Package layout + +- `mlir/dialects//` is imported as `@import("mlir/dialects/")`, + main file `.zig`. Generated builders go in `ops.zig`, exposed as + `.ops`, with hand-written helpers in `.zig`. +- The C API is `capi.h` / `capi.cc`. Dialects written here use + `{Dialect,Ops,Types,Attributes,Enums}.{td,h,cpp}`; vendored + sources keep their upstream file names. +- A generated file starts with a header naming its source and saying it is + generated. The package that generates it has a `REGENERATING.md` of agent + instructions (sources, generated vs maintained files, steps, checks) that + points here for conventions. Generator scripts are not committed. + +### BUILD.bazel + +Targets in this order: `capi` (`cc_library`, deps `:dialect`, `:headers`, +`@llvm-project//mlir:CAPIIR`), the `zig_library` (public, deps `:capi`, +`//mlir`), `test` (`zig_test`, `main = "test.zig"`, plus `:capi` when +`test.zig` imports `c`), then `td_files` (`includes = ["/"]`, direct td +deps such as `@llvm-project//mlir:OpBaseTdFiles`), the `gentbl_cc_library` +targets the dialect needs (`dialect_inc_gen`, `enums_inc_gen`, +`attrs_inc_gen`, `types_inc_gen`, `ops_inc_gen`), `headers` and `dialect`. +Every dep a source includes or imports is listed directly. Format with +`./tools/buildifier.sh`. + +### C and C++ + +- The first line is an 80-column LLVM banner: + `//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===//`. + The tag is `*- C -*-` for C headers, `*- C++ -*-` for C++ headers and + `*- tablegen -*-` for `.td`; `.cc`/`.cpp`/`.inc` banners end in `---===//`. + An optional `//` description follows, then a blank line. +- The guard is the include path in upper case with `/` and `.` as `_`, no + `ZML_` prefix (`MLIR_DIALECTS_CUDA_CUDADIALECT_H`, `..._TD`), closed with + `#endif // `. +- Includes follow LLVM order: own header, package headers, `mlir/` and + `llvm/`, then system headers. `capi.h` includes `mlir-c/IR.h` (which + provides `stdbool` and `stdint`) and the upstream `mlir-c/Dialect/.h` it + extends, inside `extern "C"` under `#ifdef __cplusplus`. +- Hand-written C APIs use fixed-width integers (`uint32_t`, `int64_t`), + parameters `MlirContext context`, `MlirAttribute attr`, `MlirType type`, + and C++ locals `MLIRContext *ctx`. `capi.cc` uses `static` helpers, not + anonymous namespaces, and one static helper loads the dialect + (`loadGPU` in `gpu/capi.cc`). + +### Names and where they come from + +`` is the dialect's C prefix, the first argument of +`MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(, )`: `Cuda`, `CuteNVGPU`, +`Triton`, `ROCDL`. Everything else is named as ODS names it: + +| ODS | C API | Zig | +|---|---|---| +| `AttrDef` (class `NameAttr`) | `mlirAttributeIsAName`, `mlirNameAttrGet` | `NameAttr`, `NameAttr.get(ctx, .{ ... })` | +| `TypeDef` (class `NameType`) | `mlirTypeIsAName`, `mlirNameTypeGet` | `NameType` | +| parameter `$elementType` | `mlirNameAttrGetElementType` | field `.elementType`, accessor `elementType()` | +| list parameter `$archs` | `...GetNumArchs`, `...GetArchs(attr, pos)` | `numArchs()`, `arch(i)` | +| `EnumAttr` / `I32EnumAttr` | `...AttrGet(context, value)`, `...AttrGetValue` | Zig enum, `XAttr.get(ctx, .{ .value = ... })`, `value()` | +| op mnemonic `fast_sin.f32` | generic operation API | builder `fast_sin_f32` | + +- Predicates have no `Attr`/`Type` suffix (`mlirTypeIsACudaEvent`). +- C getters keep MLIR's `Get`. Zig accessors are nouns, as in the rest + of ZML (`value()`, `elementType()`, `numAxes()`, `axis(i)`), never + `get`-prefixed. Predicates keep `is`/`has`. +- Enum fields use the IR spelling (`@"and"`, `@"false"`). +- A builder is named after its mnemonic with dots turned into underscores, + case kept. Its parameters are the ODS operand and attribute names, and its + result types `_type` (`_types` when variadic). +- A name that is a Zig keyword, or that collides with a member, a declaration + of the file or a builder local, takes a trailing underscore (`for_`, + `return_`, `ptr_()`, `format_()`). It is quoted only when that name is taken + too (`TupleType.@"type"(i)`, next to the `type_()` upcast). +- Alternative constructors are `mlirNameAttr
Get` in C, as MLIR's + `mlirDenseElementsAttrSplatGet`, and `get(ctx, ...)` in Zig. + +### Zig API + +```zig +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const InitArgs = struct { format: ExecutableFormat }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { ... } + pub fn attribute(self: *const Self) *const mlir.Attribute { ... } + pub fn format_(self: *const Self) ExecutableFormat { ... } +}; +``` + +- Wrappers use `mlir.Methods` with `c.MlirAttribute` or `c.MlirType`, upcast + with `attribute()` or `type_()`, and build with `get(ctx, InitArgs)`, a + struct even for one field. `get` returns `mlir.Error!*const X` when the C + getter can return null, a plain pointer otherwise. +- Free helpers are `Attr(ctx, ...)` and `Type(ctx, ...)`. +- Optional builder arguments go in `opts`: generated builders declare it + inline (`opts: struct { ... }`), hand-written ones may name it `Opts`. + Its fields are the optional operands, result types and optional, defaulted + or unit attributes, each defaulting to `null` (or the ODS default), so + callers pass `.{}`. +- Sorted imports (`std` first), no unused declarations. + +### Comments + +- Porting adds no explanatory comments; doc comments on public API stay + short. +- A generated wrapper's doc comment is the IR spelling, a colon and the ODS + summary (``/// `#cuda.executable`: Format of a compiled executable``); a + generated builder's is the IR name, a period, then the summary. + +### Tests + +Beyond the guaranteed `refAllDecls` and registration tests: a +`testContext()` that registers the package and its dependencies and calls +`mlir.Context.init(.{ .registry = registry, .threading = false })` and +`loadAllAvailableDialects()`; attributes and types round-tripped (build with +`get`, print with `{f}`, reparse, compare with `eql`); operations built and +verified. + +## Exceptions + +- Generated `.td` files start with a "generated" header instead of a banner. +- Generated C APIs keep ODS's integer types (`int`) and repeat the null check + and dialect load in every constructor. +- `cute_ir/` and `cute_nvgpu/` keep their generated builders in `.zig`; + `llvm/` keeps its builders in a private `ops.zig`. +- `cuda/`, `cute_nvgpu/` and `nvvm/` are regenerated through + `cute_ir/REGENERATING.md`, `fly_rocdl/` through `fly/REGENERATING.md`. +- `cute_ir:capi` and `cute_nvgpu:capi` each depend on the other's `:dialect`: + their C++ sources use each other's types. +- `cf`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no `capi` of + their own and bind upstream's C API target directly; `llvm`'s `capi` only + adds what upstream's leaves out (loop annotations). +- Hand-written builders with many parameters take an `Args` struct + (`func.func`, `gpu.func`, `ttir.func`, `cuda.kernel`, `cuda_tile.entry`, + `air.kernel`, `memref.subview`, `vector.transfer_read`, ...); + function-like ones carry `location` inside it. +- Some hand-written `Opts` hold required or non-null defaulted fields where + named arguments read better (`stablehlo.PadOpts`, + `mosaic_tpu.LoadOpts.sublane_mask`, `affine.ForOpts.step`). +- stablehlo, shardy, cuda_tile and ttir getters return plain pointers (their + C calls cannot fail); `ttir.pointerType` and `ttir.tensorDescType` take no + `ctx` (the C API derives it from the element type). +- Enum cases whose IR spelling is not an identifier use the C++ symbol + (rocdl's `volatile_op`), as do the generated cute_nvgpu and ttir enums. +- fly's attributes and cute_ir's algebra attributes have named constructors + (`getStatic`, `getDynamic`, `getTuple`, ...); fly's generated builders use + snake_case parameters. +- tcl mirrors the Furiosa plugin's file names and C++ style. One + `graph(op)` builder covers the 17 `tcl.graph.*` operations that share a + shape, and `loop`, `yield` and `copy` build `tcl.graph.for`, `yield` and + `copy`. Its `Tactic` and `ReduceMode` enums map to string attribute values, + its C API covers what ZML emits, and its operations are verified through + `dsl/tcl`'s tests. + +## Porting checklist + +1. Choose ``, the C prefix and the handle; create the package with the + BUILD targets above. +2. Vendor or write the `.td`/C++ sources and list every `.td` in `td_files`. +3. Write `capi.h`/`capi.cc` for registration and whatever upstream's C API + lacks: per attribute and type, a predicate, a `Get` and the parameter + getters. +4. Bind in `.zig`, or generate `ops.zig`: `registerDialect`, enums, + wrappers, helpers, builders. +5. Write `test.zig`, with the `operation_names` list. +6. If anything is generated, add a `REGENERATING.md`. +7. Make consumers depend on `//mlir/dialects/` and call + `.registerDialect(registry)`. +8. Run the verification below. + +## Verification + +Run from the repository root, with `ZIG` set to the Bazel toolchain's `zig` +(`AGENTS.md` shows how to find its standard library; `zig` is next to it). + +```bash +#!/usr/bin/env bash +d=mlir/dialects/ +bazel test "//$d:test" && bazel test //mlir/... && bazel build //... +./tools/buildifier.sh -mode=check -r "$PWD/$d" +"$ZIG" fmt --check "$d" +src=(); while IFS= read -r f; do src+=("$f"); done < <(find "$d" -maxdepth 1 \( -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.td' -o -name '*.inc' \)) +cc=(); for f in "${src[@]}"; do [[ $f == *.td || $f == *.inc ]] || cc+=("$f"); done +[ ${#cc[@]} -eq 0 ] || clang-format --dry-run --Werror --style=file:mlir/.clang-format "${cc[@]}" + +# Conformance: prints nothing on a conforming package. +for f in "${src[@]}"; do l=$(head -1 "$f"); [ ${#l} -eq 80 ] || grep -q generated <<<"$l" || echo "banner: $f"; done +grep -rln --include='*.h' --include='*.td' 'ZML_' "$d" +grep -nE 'alwayslink|glob\(' "$d/BUILD.bazel" +[ -f "$d/capi.cc" ] && grep -nE 'extern "C"|^namespace \{' "$d/capi.cc" +[ -f "$d/capi.h" ] && grep -nw unsigned "$d/capi.h" +grep -rnE --include='*.zig' 'pub fn get[A-Z][A-Za-z]*\(self' "$d" +grep -rnE --include='*.zig' --exclude=test.zig '(Type|Attribute)\.parse\(' "$d" +grep -rln --include='*.zig' --exclude=test.zig '^test ' "$d" +grep -rqF --include='*.zig' 'pub fn registerDialect' "$d" || echo "missing: registerDialect" +if grep -rqF --include='*.zig' '.make(' "$d"; then + grep -qF 'const operation_names' "$d/test.zig" || echo "missing: operation_names" + grep -qF 'test "every bound operation is registered"' "$d/test.zig" || echo "missing: registration test" +fi +git grep -n 'DialectHandle.fromString' -- dsl zml examples +exit 0 +``` diff --git a/mlir/dialects/affine/BUILD.bazel b/mlir/dialects/affine/BUILD.bazel new file mode 100644 index 00000000..06f31849 --- /dev/null +++ b/mlir/dialects/affine/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:AffineDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "affine", + import_name = "mlir/dialects/affine", + main = "affine.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":affine", + "//mlir", + ], +) diff --git a/mlir/dialects/affine/affine.zig b/mlir/dialects/affine/affine.zig new file mode 100644 index 00000000..c808619c --- /dev/null +++ b/mlir/dialects/affine/affine.zig @@ -0,0 +1,413 @@ +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("affine"); +} + +// ============================================================================= +// affine.apply / affine.min / affine.max +// ============================================================================= + +/// affine.apply — apply a 1-result affine map to a list of dim+symbol +/// operands. Result is `index`. +pub fn apply( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.apply", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +/// affine.min — minimum of all results of `map` evaluated at `map_operands`. +pub fn min( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.min", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +/// affine.max — maximum of all results of `map`. +pub fn max( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.max", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.for / affine.yield +// ============================================================================= + +pub const ForOpts = struct { + /// Constant lower bound. Pass `null` when supplying a `lower_bound_map`. + lower_bound: ?i64 = null, + /// Constant upper bound. Pass `null` when supplying an `upper_bound_map`. + upper_bound: ?i64 = null, + /// Optional map-based bounds. When non-null they take precedence over the + /// constant counterpart. + lower_bound_map: ?*const mlir.AffineMap = null, + upper_bound_map: ?*const mlir.AffineMap = null, + /// Operands feeding `lower_bound_map` (in dim+symbol order). + lower_bound_operands: []const *const mlir.Value = &.{}, + /// Operands feeding `upper_bound_map`. + upper_bound_operands: []const *const mlir.Value = &.{}, + /// Loop step (positive, defaults to 1). + step: i64 = 1, + /// Loop-carried initial values. + inits: []const *const mlir.Value = &.{}, +}; + +/// affine.for — `for %i = lower to upper step S iter_args(...)`. The body +/// block takes `index, *init_arg_types` and must terminate with `affine.yield`. +pub fn for_( + ctx: *mlir.Context, + body: *mlir.Block, + opts: ForOpts, + location: *const mlir.Location, +) *mlir.Operation { + // Resolve final maps. + const lb_map = opts.lower_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.lower_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + const ub_map = opts.upper_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.upper_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(opts.lower_bound_operands); + operands.appendSliceAssumeCapacity(opts.upper_bound_operands); + operands.appendSliceAssumeCapacity(opts.inits); + + const seg_sizes = [3]i32{ + @intCast(opts.lower_bound_operands.len), + @intCast(opts.upper_bound_operands.len), + @intCast(opts.inits.len), + }; + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (opts.inits) |v| result_types.appendAssumeCapacity(v.type_()); + + return mlir.Operation.make(ctx, "affine.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "lowerBoundMap", .affineMap(lb_map)), + .named(ctx, "upperBoundMap", .affineMap(ub_map)), + .named(ctx, "step", .int(ctx, .i64, opts.step)), + }, + // The for body may reference outer SSA values that only become + // reachable once it's appended to its parent — defer to module-level + // verify (same trick used by scf.for). + .verify = false, + .location = location, + }); +} + +/// affine.yield — terminator for affine.for / affine.if / affine.parallel. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.if +// ============================================================================= + +/// affine.if — `if condition[ ... ] { then } else { else }`. `condition` is an +/// IntegerSetAttribute (a conjunction of affine constraints). The result types +/// are inferred from the yielded values; pass `&.{}` for the no-result form. +pub fn if_( + ctx: *mlir.Context, + condition: *const mlir.IntegerSet, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var state: mlir.OperationState = .init("affine.if", location); + state.addOperands(operands); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "condition", .integerSet(condition)), + }); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create affine.if"); +} + +// ============================================================================= +// affine.load / affine.store / affine.vector_load / affine.vector_store +// ============================================================================= + +/// affine.load — load with affine indexing. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn vector_load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.parallel +// ============================================================================= + +pub const ParallelArgs = struct { + /// Per-loop-dim reductions, one entry per result. Use the same kinds as + /// `AtomicRMWKind` in `mlir/dialects/memref` (the dialect uses + /// `arith::AtomicRMWKind`). Pass `&.{}` for parallel loops with no + /// loop-carried reductions. + reductions: []const i64 = &.{}, + /// Map producing all lower-bound values; a single dim's bound is the + /// max of the results in its `lower_bounds_groups[i]` group. + lower_bounds_map: *const mlir.AffineMap, + /// One i32 per loop dim — number of map results that contribute to the + /// lower bound of that dim. Sum must equal `lower_bounds_map.numResults()`. + lower_bounds_groups: []const i32, + upper_bounds_map: *const mlir.AffineMap, + upper_bounds_groups: []const i32, + /// Per-dim integer steps (default 1). + steps: []const i64, + /// Operands feeding both bound maps. + map_operands: []const *const mlir.Value = &.{}, +}; + +/// affine.parallel — multi-dim parallel band. `body_block` takes one `index` +/// argument per loop dim and must terminate with `affine.yield`. +pub fn parallel( + ctx: *mlir.Context, + args: ParallelArgs, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // I32ElementsAttr — dense<[...]> : tensor. + const lb_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.lower_bounds_groups.len)}, + .int(ctx, .i32), + null, + ).shaped(); + const ub_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.upper_bounds_groups.len)}, + .int(ctx, .i32), + null, + ).shaped(); + + // I64SmallVectorArrayAttr — built as ArrayAttr>. + var steps_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.steps) |s| { + steps_attrs.appendAssumeCapacity(.int(ctx, .i64, s)); + } + + // Reductions are AtomicRMWKindAttr (I64EnumAttr). We let callers pass + // raw int values to avoid coupling against the enum table. + var reductions_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.reductions) |k| { + reductions_attrs.appendAssumeCapacity(.int(ctx, .i64, k)); + } + + return mlir.Operation.make(ctx, "affine.parallel", .{ + .operands = .{ .flat = args.map_operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "reductions", .array(ctx, reductions_attrs.constSlice())), + .named(ctx, "lowerBoundsMap", .affineMap(args.lower_bounds_map)), + .named(ctx, "lowerBoundsGroups", .denseElements(lb_groups_ty, args.lower_bounds_groups)), + .named(ctx, "upperBoundsMap", .affineMap(args.upper_bounds_map)), + .named(ctx, "upperBoundsGroups", .denseElements(ub_groups_ty, args.upper_bounds_groups)), + .named(ctx, "steps", .array(ctx, steps_attrs.constSlice())), + }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.delinearize_index / affine.linearize_index +// ============================================================================= + +pub const DelinearizeArgs = struct { + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, +}; + +pub fn delinearize_index( + ctx: *mlir.Context, + linear_index: *const mlir.Value, + args: DelinearizeArgs, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(linear_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + return mlir.Operation.make(ctx, "affine.delinearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), + }, + .location = location, + }); +} + +pub const LinearizeArgs = struct { + multi_index: []const *const mlir.Value, + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, + disjoint: bool = false, +}; + +pub fn linearize_index( + ctx: *mlir.Context, + args: LinearizeArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(args.multi_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + const seg_sizes = [2]i32{ + @intCast(args.multi_index.len), + @intCast(args.dynamic_basis.len), + }; + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), + }); + if (args.disjoint) { + attrs.appendAssumeCapacity(.named(ctx, "disjoint", .unit(ctx))); + } + + return mlir.Operation.make(ctx, "affine.linearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/affine/capi.cc b/mlir/dialects/affine/capi.cc new file mode 100644 index 00000000..2110365f --- /dev/null +++ b/mlir/dialects/affine/capi.cc @@ -0,0 +1,9 @@ +//===- capi.cc - C API for the affine dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/Affine/IR/AffineOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, + mlir::affine::AffineDialect) diff --git a/mlir/dialects/affine/capi.h b/mlir/dialects/affine/capi.h new file mode 100644 index 00000000..5ae66467 --- /dev/null +++ b/mlir/dialects/affine/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the affine dialect ----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AFFINE_CAPI_H +#define MLIR_DIALECTS_AFFINE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AFFINE_CAPI_H diff --git a/mlir/dialects/affine/test.zig b/mlir/dialects/affine/test.zig new file mode 100644 index 00000000..4248d4c3 --- /dev/null +++ b/mlir/dialects/affine/test.zig @@ -0,0 +1,40 @@ +const std = @import("std"); + +const affine = @import("mlir/dialects/affine"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "affine.apply", + "affine.delinearize_index", + "affine.for", + "affine.if", + "affine.linearize_index", + "affine.load", + "affine.max", + "affine.min", + "affine.parallel", + "affine.store", + "affine.vector_load", + "affine.vector_store", + "affine.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(affine); + inline for (comptime std.meta.declarations(affine)) |decl| { + const T = @field(affine, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + affine.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/air/AIRAttributes.cpp b/mlir/dialects/air/AIRAttributes.cpp new file mode 100644 index 00000000..287cb26b --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.cpp @@ -0,0 +1,140 @@ +//===- AIRAttributes.cpp - Metal AIR attributes ---------------------------===// + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIREnums.cpp.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + +void AIRDialect::registerAttributes() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + >(); +} + +static bool isValidName(StringRef name) { + return !name.empty() && !name.contains('\0'); +} + +// The size of a Metal scalar or vector, where 3-element vectors take the +// space of 4. A bool vector is an i8 vector in memory, as LLVM packs i1 +// vectors into bits. +static std::optional metalSize(Type type) { + uint64_t elements = 1; + if (auto vector = dyn_cast(type)) { + if (vector.isScalable() || vector.getRank() != 1 || + vector.getElementType().isInteger(1)) + return std::nullopt; + elements = vector.getNumElements(); + if (elements < 2 || elements > 4) + return std::nullopt; + if (elements == 3) + elements = 4; + type = vector.getElementType(); + } + // bool, char, short, int, long, half, bfloat and float. + if (!type.isInteger(1) && !type.isInteger(8) && !type.isInteger(16) && + !type.isInteger(32) && !type.isInteger(64) && !type.isF16() && + !type.isBF16() && !type.isF32()) + return std::nullopt; + return elements * llvm::divideCeil(type.getIntOrFloatBitWidth(), 8); +} + +LogicalResult +StructFieldAttr::verify(llvm::function_ref emitError, + uint64_t offset, uint64_t size, uint64_t count, + StringRef typeName, StringRef name) { + if (!size) + return emitError() << "expected a positive struct field size"; + if (!isValidName(typeName) || !isValidName(name)) + return emitError() << "expected nonempty field and type names"; + return success(); +} + +static LogicalResult +verifyStructFields(llvm::function_ref emitError, + LLVM::LLVMStructType type, uint64_t typeSize, + ArrayRef fields) { + if (type.isOpaque() || type.isPacked()) + return emitError() << "expected a sized, unpacked struct"; + ArrayRef body = type.getBody(); + if (body.size() != fields.size()) + return emitError() << "expected one field per struct element"; + uint64_t end = 0; + for (auto [member, field] : llvm::zip_equal(body, fields)) { + Type element = member; + uint64_t count = 0; + if (auto array = dyn_cast(element)) { + count = array.getNumElements(); + element = array.getElementType(); + } + std::optional size = metalSize(element); + if (!size || *size != field.getSize() || count != field.getCount()) + return emitError() << "field " << field.getName() + << " does not match the struct element " << element; + if (field.getOffset() < end || field.getOffset() % *size != 0) + return emitError() << "field " << field.getName() + << " is misplaced at offset " << field.getOffset(); + end = field.getOffset() + *size * std::max(count, 1); + } + if (end > typeSize) + return emitError() << "struct fields overflow the buffer type size"; + return success(); +} + +LogicalResult BufferAttr::verify( + llvm::function_ref emitError, unsigned locationIndex, + AddressSpace addressSpace, Access access, Type elementType, + StringRef typeName, uint64_t typeSize, uint64_t typeAlignment, + uint64_t bufferSize, bool noalias, ArrayRef fields) { + if (addressSpace == AddressSpace::thread) + return emitError() << "buffers are not in thread memory"; + if (addressSpace == AddressSpace::constant && access != Access::read) + return emitError() << "constant buffers are read-only"; + if (addressSpace == AddressSpace::threadgroup && access != Access::read_write) + return emitError() << "threadgroup buffers are read_write"; + if (!elementType) + return emitError() << "expected a buffer element type"; + if (!isValidName(typeName)) + return emitError() << "expected a nonempty Metal type name"; + if (!typeSize || !llvm::isPowerOf2_64(typeAlignment) || + typeSize % typeAlignment != 0) + return emitError() << "expected a positive ABI size divisible by a " + "power-of-two alignment"; + if (auto structType = dyn_cast(elementType)) { + if (addressSpace != AddressSpace::constant) + return emitError() << "struct buffers must be constant"; + if (failed(verifyStructFields(emitError, structType, typeSize, fields))) + return failure(); + } else { + std::optional size = metalSize(elementType); + if (!size) + return emitError() << "expected a scalar, vector or struct element type"; + if (*size != typeSize) + return emitError() << "ABI size " << typeSize << " does not match " + << elementType; + if (!fields.empty()) + return emitError() << "only struct buffers have fields"; + } + if (bufferSize && (addressSpace != AddressSpace::constant || + bufferSize < typeSize || bufferSize % typeAlignment)) + return emitError() << "only constant buffers have a buffer size, which " + "must cover the element and respect its alignment"; + return success(); +} diff --git a/mlir/dialects/air/AIRAttributes.h b/mlir/dialects/air/AIRAttributes.h new file mode 100644 index 00000000..4a31eb77 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.h @@ -0,0 +1,12 @@ +//===- AIRAttributes.h - Metal AIR attributes -------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_H +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_H + +#include "mlir/dialects/air/AIREnums.h" + +#include "mlir/IR/Attributes.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_H diff --git a/mlir/dialects/air/AIRAttributes.td b/mlir/dialects/air/AIRAttributes.td new file mode 100644 index 00000000..3e4852d0 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.td @@ -0,0 +1,66 @@ +//===- AIRAttributes.td - Metal AIR attributes -------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD + +include "mlir/dialects/air/AIRDialect.td" +include "mlir/dialects/air/AIREnums.td" + +def AIR_BuiltinAttr : EnumAttr { + let summary = "Kernel argument builtin"; + let description = [{ + Attach as `air.builtin` to a kernel argument. The position and size + builtins are `uint` or `ushort` scalars or 2- and 3-element vectors + (`i32`, `i16`, `vector<3xi32>`, ...); the index builtins are `uint` or + `ushort` scalars. + }]; +} + +def AIR_MemFlagsAttr : EnumAttr { + let summary = "Memory a barrier orders"; +} + +def AIR_StructField : AttrDef { + let mnemonic = "struct_field"; + let summary = "Field of a constant buffer struct"; + let description = [{ + One `air.struct_type_info` entry: the byte offset, the element size, + the array length (zero for a non-array field), the Metal type name and + the field name. + }]; + let parameters = (ins "uint64_t":$offset, + "uint64_t":$size, + DefaultValuedParameter<"uint64_t", "0">:$count, + StringRefParameter<>:$typeName, + StringRefParameter<>:$name); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +def AIR_Buffer : AttrDef { + let mnemonic = "buffer"; + let summary = "Kernel buffer argument"; + let description = [{ + Attach as `air.buffer` to a kernel argument of type `!llvm.ptr` in + `addressSpace`. `elementType` is the pointee (a scalar, a vector, or an + `!llvm.struct` described by `fields`), `typeName` its Metal spelling, + and `typeSize` and `typeAlignment` its size and alignment in bytes. + `locationIndex` is the `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + A constant reference records its size as `bufferSize`. `noalias` is a + `__restrict__` pointer. + }]; + let parameters = (ins "unsigned":$locationIndex, + EnumParameter:$addressSpace, + EnumParameter:$access, + "::mlir::Type":$elementType, + StringRefParameter<>:$typeName, + "uint64_t":$typeSize, + "uint64_t":$typeAlignment, + DefaultValuedParameter<"uint64_t", "0">:$bufferSize, + DefaultValuedParameter<"bool", "false">:$noalias, + OptionalArrayRefParameter<"StructFieldAttr">:$fields); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD diff --git a/mlir/dialects/air/AIRDialect.cpp b/mlir/dialects/air/AIRDialect.cpp new file mode 100644 index 00000000..7ad56427 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.cpp @@ -0,0 +1,168 @@ +//===- AIRDialect.cpp - Metal AIR dialect ---------------------------------===// + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/Interfaces/FunctionInterfaces.h" +#include "llvm/ADT/DenseSet.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIRDialect.cpp.inc" + +void AIRDialect::initialize() { + registerAttributes(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/air/AIROps.cpp.inc" + >(); +} + +static bool isKernelArgumentAttribute(StringRef name) { + return name == AIRDialect::getBuiltinAttrName() || + name == AIRDialect::getBufferAttrName() || + name == AIRDialect::getArgNameAttrName(); +} + +// Position and size builtins may be 2- or 3-element vectors; index builtins +// are scalars. +static bool isVectorBuiltin(Builtin builtin) { + switch (builtin) { + case Builtin::thread_position_in_grid: + case Builtin::threadgroup_position_in_grid: + case Builtin::thread_position_in_threadgroup: + case Builtin::threads_per_threadgroup: + case Builtin::threadgroups_per_grid: + case Builtin::threads_per_grid: + return true; + default: + return false; + } +} + +static LogicalResult verifyKernelArgument(FunctionOpInterface function, + unsigned index) { + Operation *op = function.getOperation(); + Attribute builtin = + function.getArgAttr(index, AIRDialect::getBuiltinAttrName()); + Attribute buffer = + function.getArgAttr(index, AIRDialect::getBufferAttrName()); + if (bool(builtin) == bool(buffer)) + return op->emitOpError() + << "argument " << index + << " must have exactly one of air.builtin or air.buffer"; + if (Attribute name = + function.getArgAttr(index, AIRDialect::getArgNameAttrName())) + if (auto string = dyn_cast(name); + !string || string.getValue().empty()) + return op->emitOpError("air.arg_name must be a nonempty string"); + Type type = function.getArgumentTypes()[index]; + if (builtin) { + auto metadata = dyn_cast(builtin); + if (!metadata) + return op->emitOpError("air.builtin must be an #air.builtin"); + if (auto vector = dyn_cast(type)) { + if (!isVectorBuiltin(metadata.getValue()) || vector.isScalable() || + vector.getRank() != 1 || + (vector.getNumElements() != 2 && vector.getNumElements() != 3)) + return op->emitOpError() + << "builtin " << stringifyBuiltin(metadata.getValue()) + << " cannot be a " << type; + type = vector.getElementType(); + } + if (!type.isSignlessInteger(32) && !type.isSignlessInteger(16)) + return op->emitOpError("AIR builtins are uint or ushort scalars or " + "vectors"); + return success(); + } + auto metadata = dyn_cast(buffer); + if (!metadata) + return op->emitOpError("air.buffer must be an #air.buffer"); + auto pointer = dyn_cast(type); + if (!pointer || + pointer.getAddressSpace() != unsigned(metadata.getAddressSpace())) + return op->emitOpError() + << "argument " << index + << " must be an !llvm.ptr in the address space of its air.buffer"; + return success(); +} + +LogicalResult AIRDialect::verifyOperationAttribute(Operation *op, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " belongs on a kernel argument"; + if (name == AIRDialect::getFastMathEnableAttrName()) { + if (!isa(op) || !isa(attr.getValue())) + return op->emitOpError() << name << " is a unit attribute of a module"; + return success(); + } + if (name != AIRDialect::getKernelAttrName()) + return op->emitOpError() << "unknown AIR attribute " << name; + auto function = dyn_cast(op); + if (!function || !isa(attr.getValue())) + return op->emitOpError( + "air.kernel requires a function and a unit attribute"); + if (failed(op->getName().verifyInvariants(op))) + return failure(); + if (function.isExternal()) + return op->emitOpError("AIR kernels need a body"); + if (function.getNumResults() != 0) + return op->emitOpError("AIR kernels cannot return values"); + if (auto llvmFunction = dyn_cast(op)) + if (llvmFunction.isVarArg()) + return op->emitOpError("AIR kernels cannot be variadic"); + llvm::DenseSet builtins; + llvm::DenseSet> bindings; + for (unsigned i = 0; i < function.getNumArguments(); ++i) { + if (failed(verifyKernelArgument(function, i))) + return failure(); + if (auto builtin = function.getArgAttrOfType( + i, AIRDialect::getBuiltinAttrName())) { + if (!builtins.insert(builtin.getValue()).second) + return op->emitOpError() << "duplicate AIR builtin " + << stringifyBuiltin(builtin.getValue()); + continue; + } + auto buffer = function.getArgAttrOfType( + i, AIRDialect::getBufferAttrName()); + // Device and constant buffers share bindings; threadgroup buffers do not. + auto key = std::make_pair( + unsigned(buffer.getAddressSpace() == AddressSpace::threadgroup), + buffer.getLocationIndex()); + if (!bindings.insert(key).second) + return op->emitOpError() + << "duplicate AIR buffer binding " << buffer.getLocationIndex(); + } + return success(); +} + +LogicalResult AIRDialect::verifyRegionArgAttribute(Operation *op, + unsigned regionIndex, + unsigned argIndex, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (!isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " is not an AIR argument attribute"; + auto function = dyn_cast(op); + if (!function || regionIndex != 0 || + !op->getAttrOfType(AIRDialect::getKernelAttrName())) + return op->emitOpError() + << name << " requires an air.kernel function argument"; + return verifyKernelArgument(function, argIndex); +} + +LogicalResult AIRDialect::verifyRegionResultAttribute(Operation *op, unsigned, + unsigned, + NamedAttribute attr) { + return op->emitOpError() << attr.getName() + << " cannot annotate a function result"; +} diff --git a/mlir/dialects/air/AIRDialect.h b/mlir/dialects/air/AIRDialect.h new file mode 100644 index 00000000..e28ec0bc --- /dev/null +++ b/mlir/dialects/air/AIRDialect.h @@ -0,0 +1,9 @@ +//===- AIRDialect.h - Metal AIR dialect -------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_H +#define MLIR_DIALECTS_AIR_AIRDIALECT_H + +#include "mlir/IR/Dialect.h" +#include "mlir/dialects/air/AIRDialect.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_H diff --git a/mlir/dialects/air/AIRDialect.td b/mlir/dialects/air/AIRDialect.td new file mode 100644 index 00000000..96264d76 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.td @@ -0,0 +1,53 @@ +//===- AIRDialect.td - Metal AIR dialect -------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_TD +#define MLIR_DIALECTS_AIR_AIRDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/OpBase.td" + +def AIR_Dialect : Dialect { + let name = "air"; + let cppNamespace = "::mlir::metal::air"; + let summary = "Apple Metal AIR intrinsics and kernel ABI metadata"; + let description = [{ + The Metal-specific part of the LLVM IR Apple's Metal compiler emits (AIR), + not AMD's asynchronous AIR dialect. Each operation is an `air.*` + intrinsic named as Apple names it, without the type suffix its operand + and result types determine: `air.fast_exp %x : f32` is + `@air.fast_exp.f32`, `air.max.s %a, %b : vector<2xi32>` is + `@air.max.s.v2i32`. Kernels are `func.func` with `air.kernel`, and their + arguments carry the `!air.kernel` descriptors as `air.builtin` or + `air.buffer`. Arithmetic, memory and control flow use the arith, llvm + and scf dialects; pointers are `!llvm.ptr` in AIR's address spaces. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; + let hasOperationAttrVerify = 1; + let hasRegionArgAttrVerify = 1; + let hasRegionResultAttrVerify = 1; + let extraClassDeclaration = [{ + void registerAttributes(); + + static constexpr ::llvm::StringLiteral getKernelAttrName() { + return ::llvm::StringLiteral("air.kernel"); + } + static constexpr ::llvm::StringLiteral getBuiltinAttrName() { + return ::llvm::StringLiteral("air.builtin"); + } + static constexpr ::llvm::StringLiteral getBufferAttrName() { + return ::llvm::StringLiteral("air.buffer"); + } + static constexpr ::llvm::StringLiteral getArgNameAttrName() { + return ::llvm::StringLiteral("air.arg_name"); + } + static constexpr ::llvm::StringLiteral getFastMathEnableAttrName() { + return ::llvm::StringLiteral("air.fast_math_enable"); + } + }]; +} + +class AIR_Op traits = []> + : Op; + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_TD diff --git a/mlir/dialects/air/AIREnums.h b/mlir/dialects/air/AIREnums.h new file mode 100644 index 00000000..385f2041 --- /dev/null +++ b/mlir/dialects/air/AIREnums.h @@ -0,0 +1,10 @@ +//===- AIREnums.h - Metal AIR enums -----------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_H +#define MLIR_DIALECTS_AIR_AIRENUMS_H + +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/dialects/air/AIREnums.h.inc" +#include "llvm/ADT/StringExtras.h" + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_H diff --git a/mlir/dialects/air/AIREnums.td b/mlir/dialects/air/AIREnums.td new file mode 100644 index 00000000..61e9c25a --- /dev/null +++ b/mlir/dialects/air/AIREnums.td @@ -0,0 +1,54 @@ +//===- AIREnums.td - Metal AIR enums -----------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_TD +#define MLIR_DIALECTS_AIR_AIRENUMS_TD + +include "mlir/IR/EnumAttr.td" + +def AIR_Builtin : I32EnumAttr<"Builtin", "Kernel argument builtin", [ + I32EnumAttrCase<"thread_position_in_grid", 0>, + I32EnumAttrCase<"threadgroup_position_in_grid", 1>, + I32EnumAttrCase<"thread_position_in_threadgroup", 2>, + I32EnumAttrCase<"threads_per_threadgroup", 3>, + I32EnumAttrCase<"threadgroups_per_grid", 4>, + I32EnumAttrCase<"threads_per_grid", 5>, + I32EnumAttrCase<"thread_index_in_simdgroup", 6>, + I32EnumAttrCase<"simdgroup_index_in_threadgroup", 7>, + I32EnumAttrCase<"threads_per_simdgroup", 8>, + I32EnumAttrCase<"simdgroups_per_threadgroup", 9>, + I32EnumAttrCase<"thread_index_in_threadgroup", 10>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Address space", [ + I32EnumAttrCase<"thread", 0>, + I32EnumAttrCase<"device", 1>, + I32EnumAttrCase<"constant", 2>, + I32EnumAttrCase<"threadgroup", 3>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_Access : I32EnumAttr<"Access", "Buffer access", [ + I32EnumAttrCase<"read", 0>, + I32EnumAttrCase<"read_write", 1>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_MemFlags : I32BitEnumAttr<"MemFlags", "Barrier memory flags", [ + I32BitEnumAttrCaseNone<"none">, + I32BitEnumAttrCaseBit<"device", 0>, + I32BitEnumAttrCaseBit<"threadgroup", 1>, + I32BitEnumAttrCaseBit<"texture", 2>, +]> { + let separator = "|"; + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_TD diff --git a/mlir/dialects/air/AIROps.cpp b/mlir/dialects/air/AIROps.cpp new file mode 100644 index 00000000..85e48837 --- /dev/null +++ b/mlir/dialects/air/AIROps.cpp @@ -0,0 +1,64 @@ +//===- AIROps.cpp - Metal AIR operations ----------------------------------===// + +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/IR/TypeUtilities.h" + +using namespace mlir; +using namespace mlir::metal::air; + +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.cpp.inc" + +LogicalResult DotOp::verify() { + if (getElementTypeOrSelf(getX().getType()) != getResult().getType()) + return emitOpError("expects the operands' element type as result"); + return success(); +} + +static LogicalResult verifyConvert(Operation *op, bool toOrFromInteger) { + Type input = op->getOperand(0).getType(); + Type result = op->getResult(0).getType(); + if (failed(verifyCompatibleShape(input, result)) || + isa(input) != isa(result)) + return op->emitOpError("expects operands and results of the same shape"); + bool inputInt = isa(getElementTypeOrSelf(input)); + bool resultInt = isa(getElementTypeOrSelf(result)); + if (!toOrFromInteger && (inputInt || resultInt || input == result)) + return op->emitOpError("converts between distinct float types"); + if (!toOrFromInteger && !isa(input) && + (input.isF32() || result.isF32())) + return op->emitOpError("scalar f32 conversions are LLVM casts in Apple's " + "compiler"); + if (toOrFromInteger && inputInt == resultInt) + return op->emitOpError("converts between an integer and a float type"); + if (resultInt && cast(op) + .getFastMathFlagsAttr() + .getValue() != arith::FastMathFlags::none) + return op->emitOpError("integer results take no fastmath flags"); + return success(); +} + +LogicalResult ConvertOp::verify() { return verifyConvert(*this, false); } +LogicalResult ConvertSOp::verify() { return verifyConvert(*this, true); } +LogicalResult ConvertUOp::verify() { return verifyConvert(*this, true); } + +LogicalResult SimdgroupMatrixMultiplyAccumulateOp::verify() { + auto element = [](Value value) { + return cast(value.getType()).getElementType(); + }; + Type d = element(getResult()), a = element(getA()), b = element(getB()), + c = element(getC()); + // The combinations Apple's compiler emits for simdgroup_multiply_accumulate. + bool mixed = d.isF32() && c.isF32() && (a.isF32() || a == b) && + (b.isF16() || b.isBF16() || b.isF32()); + bool half = d.isF16() && a.isF16() && b.isF16() && c.isF16(); + if (!mixed && !half) + return emitOpError("unsupported element types ") + << a << " * " << b << " + " << c << " -> " << d; + return success(); +} diff --git a/mlir/dialects/air/AIROps.h b/mlir/dialects/air/AIROps.h new file mode 100644 index 00000000..baa5a517 --- /dev/null +++ b/mlir/dialects/air/AIROps.h @@ -0,0 +1,16 @@ +//===- AIROps.h - Metal AIR operations --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_H +#define MLIR_DIALECTS_AIR_AIROPS_H + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIROPS_H diff --git a/mlir/dialects/air/AIROps.td b/mlir/dialects/air/AIROps.td new file mode 100644 index 00000000..6d072437 --- /dev/null +++ b/mlir/dialects/air/AIROps.td @@ -0,0 +1,352 @@ +//===- AIROps.td - Metal AIR operations --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_TD +#define MLIR_DIALECTS_AIR_AIROPS_TD + +include "mlir/Dialect/Arith/IR/ArithBase.td" +include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td" +include "mlir/Dialect/LLVMIR/LLVMOpBase.td" +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/air/AIRAttributes.td" + +class AIR_VectorOf types> + : FixedVectorOfRankAndLengthAndType<[1], [2, 3, 4], types>; +class AIR_ScalarOrVectorOf types> + : AnyTypeOf])>; + +def AIR_FloatLike : AIR_ScalarOrVectorOf<[F16, F32]>; +def AIR_F32Like : AIR_ScalarOrVectorOf<[F32]>; +def AIR_IntLike : AIR_ScalarOrVectorOf<[I8, I16, I32, I64]>; +def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I8, I16, I32]>; +def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I8, I16, I32, I64]>; +def AIR_Matrix : FixedVectorOfRankAndLengthAndType<[1], [64], [F16, BF16, F32]>; +def AIR_MatrixVector : FixedVectorOfShapeAndType<[2], I64>; +def AIR_MatrixPointer : AnyTypeOf<[LLVM_PointerInAddressSpace<1>, + LLVM_PointerInAddressSpace<3>]>; + +def AIR_FastMath : DefaultValuedAttr; + +//===----------------------------------------------------------------------===// +// Math +//===----------------------------------------------------------------------===// + +class AIR_ElementwiseOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` on each element"; + let description = "Calls `@air." # mnemonic # ".`."; +} + +class AIR_FloatUnaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatBinaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatTernaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, type:$z, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = [{ + $x `,` $y `,` $z (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_IntUnaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_IntBinaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y attr-dict `:` type($result)"; +} + +class AIR_IntTernaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y, AIR_IntLike:$z); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y `,` $z attr-dict `:` type($result)"; +} + +// `exp` is `metal::precise::exp`; `fast_exp` is `metal::fast::exp`, which +// `exp` selects under fast math. Only f32 has fast variants. +def AIR_ExpOp : AIR_FloatUnaryOp<"exp">; +def AIR_FastExpOp : AIR_FloatUnaryOp<"fast_exp", AIR_F32Like>; +def AIR_Exp2Op : AIR_FloatUnaryOp<"exp2">; +def AIR_FastExp2Op : AIR_FloatUnaryOp<"fast_exp2", AIR_F32Like>; +def AIR_LogOp : AIR_FloatUnaryOp<"log">; +def AIR_FastLogOp : AIR_FloatUnaryOp<"fast_log", AIR_F32Like>; +def AIR_Log2Op : AIR_FloatUnaryOp<"log2">; +def AIR_FastLog2Op : AIR_FloatUnaryOp<"fast_log2", AIR_F32Like>; +def AIR_TanhOp : AIR_FloatUnaryOp<"tanh">; +def AIR_FastTanhOp : AIR_FloatUnaryOp<"fast_tanh", AIR_F32Like>; +def AIR_SqrtOp : AIR_FloatUnaryOp<"sqrt">; +def AIR_FastSqrtOp : AIR_FloatUnaryOp<"fast_sqrt", AIR_F32Like>; +def AIR_RsqrtOp : AIR_FloatUnaryOp<"rsqrt">; +def AIR_FastRsqrtOp : AIR_FloatUnaryOp<"fast_rsqrt", AIR_F32Like>; +def AIR_FabsOp : AIR_FloatUnaryOp<"fabs">; +def AIR_FastFabsOp : AIR_FloatUnaryOp<"fast_fabs", AIR_F32Like>; +def AIR_CeilOp : AIR_FloatUnaryOp<"ceil">; +def AIR_FastCeilOp : AIR_FloatUnaryOp<"fast_ceil", AIR_F32Like>; +def AIR_FloorOp : AIR_FloatUnaryOp<"floor">; +def AIR_FastFloorOp : AIR_FloatUnaryOp<"fast_floor", AIR_F32Like>; +def AIR_RoundOp : AIR_FloatUnaryOp<"round">; +def AIR_FastRoundOp : AIR_FloatUnaryOp<"fast_round", AIR_F32Like>; +def AIR_TruncOp : AIR_FloatUnaryOp<"trunc">; +def AIR_FastTruncOp : AIR_FloatUnaryOp<"fast_trunc", AIR_F32Like>; +def AIR_SinOp : AIR_FloatUnaryOp<"sin">; +def AIR_FastSinOp : AIR_FloatUnaryOp<"fast_sin", AIR_F32Like>; +def AIR_CosOp : AIR_FloatUnaryOp<"cos">; +def AIR_FastCosOp : AIR_FloatUnaryOp<"fast_cos", AIR_F32Like>; + +def AIR_FmaxOp : AIR_FloatBinaryOp<"fmax">; +def AIR_FastFmaxOp : AIR_FloatBinaryOp<"fast_fmax", AIR_F32Like>; +def AIR_FminOp : AIR_FloatBinaryOp<"fmin">; +def AIR_FastFminOp : AIR_FloatBinaryOp<"fast_fmin", AIR_F32Like>; +def AIR_PowOp : AIR_FloatBinaryOp<"pow">; +def AIR_FastPowOp : AIR_FloatBinaryOp<"fast_pow", AIR_F32Like>; +def AIR_FmodOp : AIR_FloatBinaryOp<"fmod">; +def AIR_FastFmodOp : AIR_FloatBinaryOp<"fast_fmod", AIR_F32Like>; + +def AIR_FmaOp : AIR_FloatTernaryOp<"fma">; +def AIR_ClampOp : AIR_FloatTernaryOp<"clamp">; +def AIR_FastClampOp : AIR_FloatTernaryOp<"fast_clamp", AIR_F32Like>; + +def AIR_DotOp : AIR_Op<"dot", [Pure, AllTypesMatch<["x", "y"]>, + DeclareOpInterfaceMethods]> { + let summary = "Dot product of two float vectors"; + let description = "Calls `@air.dot.`."; + let arguments = (ins AIR_VectorOf<[F16, F32]>:$x, AIR_VectorOf<[F16, F32]>:$y, + AIR_FastMath:$fastmath); + let results = (outs AnyTypeOf<[F16, F32]>:$result); + let assemblyFormat = [{ + $x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `->` + type($result) + }]; + let hasVerifier = 1; +} + +// `.s` and `.u` are the signed and unsigned overloads of MSL's integer +// functions. +def AIR_MaxSOp : AIR_IntBinaryOp<"max.s">; +def AIR_MaxUOp : AIR_IntBinaryOp<"max.u">; +def AIR_MinSOp : AIR_IntBinaryOp<"min.s">; +def AIR_MinUOp : AIR_IntBinaryOp<"min.u">; +def AIR_AbsSOp : AIR_IntUnaryOp<"abs.s">; +def AIR_ClampSOp : AIR_IntTernaryOp<"clamp.s">; +def AIR_ClampUOp : AIR_IntTernaryOp<"clamp.u">; + +//===----------------------------------------------------------------------===// +// Conversions +//===----------------------------------------------------------------------===// + +class AIR_ConvertBaseOp + : AIR_Op]> { + let summary = summaryText; + let description = [{ + Calls `@air.convert....`, + where floats are `f` and the integer side is `s` or `u` as the mnemonic + says. Scalar f32 to and from f16 or bf16, and integer to integer, are + LLVM casts in Apple's compiler. + }]; + let arguments = (ins AIR_ConvertLike:$x, AIR_FastMath:$fastmath); + let results = (outs AIR_ConvertLike:$result); + let assemblyFormat = [{ + $x (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `to` type($result) + }]; + let hasVerifier = 1; +} + +def AIR_ConvertOp : AIR_ConvertBaseOp<"convert", "Float to float conversion">; +def AIR_ConvertSOp : AIR_ConvertBaseOp<"convert.s", + "Conversion between floats and signed integers">; +def AIR_ConvertUOp : AIR_ConvertBaseOp<"convert.u", + "Conversion between floats and unsigned integers">; + +//===----------------------------------------------------------------------===// +// SIMD-group functions +//===----------------------------------------------------------------------===// + +// SIMD-group functions read the other lanes, so they are not Pure: unknown +// effects keep them from being speculated or moved across divergent control. +class AIR_SimdReduceOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = "Calls `@air." # mnemonic # ".`."; + let results = (outs type:$result); +} + +class AIR_SimdFloatReduceOp + : AIR_SimdReduceOp]> { + let arguments = (ins AIR_FloatLike:$x, AIR_FastMath:$fastmath); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_SimdIntReduceOp + : AIR_SimdReduceOp { + let arguments = (ins AIR_SimdIntLike:$x); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_SimdShuffleOp traits = []> + : AIR_Op]> { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = [{ + Calls `@air.}] # mnemonic # [{.`. `lane` is the source lane, the + XOR mask or the lane delta, as for the MSL function. + }]; + let results = (outs type:$result); +} + +class AIR_SimdFloatShuffleOp + : AIR_SimdShuffleOp]> { + let arguments = (ins AIR_FloatLike:$data, I16:$lane, AIR_FastMath:$fastmath); + let assemblyFormat = [{ + $data `,` $lane (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_SimdIntShuffleOp + : AIR_SimdShuffleOp { + let arguments = (ins AIR_SimdIntLike:$data, I16:$lane); + let assemblyFormat = "$data `,` $lane attr-dict `:` type($result)"; +} + +def AIR_SimdSumOp : AIR_SimdFloatReduceOp<"simd_sum">; +def AIR_SimdSumSOp : AIR_SimdIntReduceOp<"simd_sum.s">; +def AIR_SimdSumUOp : AIR_SimdIntReduceOp<"simd_sum.u">; +def AIR_SimdMaxOp : AIR_SimdFloatReduceOp<"simd_max">; +def AIR_SimdMaxSOp : AIR_SimdIntReduceOp<"simd_max.s">; +def AIR_SimdMaxUOp : AIR_SimdIntReduceOp<"simd_max.u">; +def AIR_SimdMinOp : AIR_SimdFloatReduceOp<"simd_min">; +def AIR_SimdMinSOp : AIR_SimdIntReduceOp<"simd_min.s">; +def AIR_SimdMinUOp : AIR_SimdIntReduceOp<"simd_min.u">; +def AIR_SimdPrefixExclusiveSumOp + : AIR_SimdFloatReduceOp<"simd_prefix_exclusive_sum">; + +def AIR_SimdShuffleOp : AIR_SimdFloatShuffleOp<"simd_shuffle">; +def AIR_SimdShuffleSOp : AIR_SimdIntShuffleOp<"simd_shuffle.s">; +def AIR_SimdShuffleUOp : AIR_SimdIntShuffleOp<"simd_shuffle.u">; +def AIR_SimdShuffleXorOp : AIR_SimdFloatShuffleOp<"simd_shuffle_xor">; +def AIR_SimdShuffleXorSOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.s">; +def AIR_SimdShuffleXorUOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.u">; +def AIR_SimdShuffleDownOp : AIR_SimdFloatShuffleOp<"simd_shuffle_down">; +def AIR_SimdShuffleDownSOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.s">; +def AIR_SimdShuffleDownUOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.u">; +def AIR_SimdBroadcastOp : AIR_SimdFloatShuffleOp<"simd_broadcast">; +def AIR_SimdBroadcastSOp : AIR_SimdIntShuffleOp<"simd_broadcast.s">; +def AIR_SimdBroadcastUOp : AIR_SimdIntShuffleOp<"simd_broadcast.u">; + +class AIR_SimdVoteOp : AIR_Op { + let summary = summaryText; + let description = "Calls `@air." # mnemonic # "`."; + let arguments = (ins I1:$x); + let results = (outs I1:$result); + let assemblyFormat = "$x attr-dict"; +} + +def AIR_SimdAnyOp : AIR_SimdVoteOp<"simd_any", + "Whether `x` is true in any active lane">; +def AIR_SimdAllOp : AIR_SimdVoteOp<"simd_all", + "Whether `x` is true in every active lane">; + +//===----------------------------------------------------------------------===// +// Barriers +//===----------------------------------------------------------------------===// + +def AIR_WgBarrierOp : AIR_Op<"wg.barrier"> { + let summary = "threadgroup_barrier"; + let description = "Calls `@air.wg.barrier(i32 mem_flags, i32 1)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +def AIR_SimdgroupBarrierOp : AIR_Op<"simdgroup.barrier"> { + let summary = "simdgroup_barrier"; + let description = "Calls `@air.simdgroup.barrier(i32 mem_flags, i32 4)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +//===----------------------------------------------------------------------===// +// SIMD-group matrices +//===----------------------------------------------------------------------===// + +// A simdgroup_matrix is a vector<64xT>; thread_elements() are its +// elements 0 and 1. +def AIR_SimdgroupMatrixLoadOp : AIR_Op<"simdgroup_matrix_8x8_load", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_load"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_load.v64.p
`. + `simdgroup_load(m, p, ld, origin, transpose)` passes `extents` (ld, 8) + and `strides` (1, ld), swapped when transposed. + }]; + let arguments = (ins AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $pointer `,` $extents `,` $strides `,` $origin (`fastmath` `` $fastmath^)? + attr-dict `:` qualified(type($pointer)) `->` type($result) + }]; +} + +def AIR_SimdgroupMatrixStoreOp : AIR_Op<"simdgroup_matrix_8x8_store"> { + let summary = "simdgroup_store"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_store.v64.p
`, with + the operands of `simdgroup_matrix_8x8_load`. + }]; + let arguments = (ins AIR_Matrix:$value, + AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin); + let assemblyFormat = [{ + $value `,` $pointer `,` $extents `,` $strides `,` $origin attr-dict `:` + type($value) `,` qualified(type($pointer)) + }]; +} + +def AIR_SimdgroupMatrixMultiplyAccumulateOp + : AIR_Op<"simdgroup_matrix_8x8_multiply_accumulate", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_multiply_accumulate"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_multiply_accumulate.v64.v64.v64.v64`: + `result = a * b + c`. `simdgroup_multiply` passes a zero `c`. + }]; + let arguments = (ins AIR_Matrix:$a, AIR_Matrix:$b, AIR_Matrix:$c, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $a `,` $b `,` $c (`fastmath` `` $fastmath^)? attr-dict `:` type($a) `,` + type($b) `,` type($c) `->` type($result) + }]; + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIROPS_TD diff --git a/mlir/dialects/air/BUILD.bazel b/mlir/dialects/air/BUILD.bazel new file mode 100644 index 00000000..9fb6ed6c --- /dev/null +++ b/mlir/dialects/air/BUILD.bazel @@ -0,0 +1,179 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "air", + import_name = "mlir/dialects/air", + main = "air.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":air", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + ], +) + +td_library( + name = "td_files", + srcs = [ + "AIRAttributes.td", + "AIRDialect.td", + "AIREnums.td", + "AIROps.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:ArithOpsInterfacesTdFiles", + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=air", + ], + "AIRDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=air", + ], + "AIRDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "AIREnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "AIREnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIREnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=air", + ], + "AIRAttributes.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=air", + ], + "AIRAttributes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRAttributes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "AIROps.h.inc", + ), + ( + ["-gen-op-defs"], + "AIROps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIROps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "AIRAttributes.h", + "AIRDialect.h", + "AIREnums.h", + "AIROps.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "AIRAttributes.cpp", + "AIRDialect.cpp", + "AIROps.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/air/README.md b/mlir/dialects/air/README.md new file mode 100644 index 00000000..cd8a0531 --- /dev/null +++ b/mlir/dialects/air/README.md @@ -0,0 +1,90 @@ +# Metal AIR dialect + +The Metal-specific part of AIR, the LLVM IR Apple's Metal compiler emits (not +AMD's asynchronous AIR dialect). Namespace `air`, C++ namespace +`mlir::metal::air`, C API prefix `mlirMetalAIR`. `dsl/metal` emits it, and +XLA's `__gpu$xla.gpu.air` custom call lowers it, from a vendored copy, to a +metallib. + +## Operations + +Each operation is an `air.*` intrinsic, named as Apple's compiler names it +minus the type suffix its operand and result types determine: + +| Operation | Intrinsic | +|---|---| +| `air.fast_exp %x : vector<2xf32>` | `@air.fast_exp.v2f32` | +| `air.max.u %a, %b : i32` | `@air.max.u.i32` | +| `air.simd_shuffle_xor %x, %lane : f16` | `@air.simd_shuffle_xor.f16` | +| `air.convert.s %x : i32 to f32` | `@air.convert.f.f32.s.i32` | +| `air.simdgroup_matrix_8x8_load %p, ... : !llvm.ptr<3> -> vector<64xbf16>` | `@air.simdgroup_matrix_8x8_load.v64bf16.p3bf16` | +| `air.wg.barrier ` | `@air.wg.barrier(i32 2, i32 1)` | + +- Float math: `exp`, `exp2`, `log`, `log2`, `tanh`, `sqrt`, `rsqrt`, `fabs`, + `ceil`, `floor`, `round`, `trunc`, `sin`, `cos`, `fmax`, `fmin`, `pow`, + `fmod`, `fma`, `clamp` and `dot` on f16 and f32. The `fast_` variants are + f32 only: under fast math (MSL's default) `exp(x)` is `fast_exp`, and + `metal::precise::exp` is `exp`. +- Integer math: `max`, `min`, `clamp` and `abs`, as `.s` and `.u` + overloads. +- SIMD-group functions: `simd_sum`, `simd_max`, `simd_min`, + `simd_prefix_exclusive_sum`, `simd_shuffle`, `simd_shuffle_xor`, + `simd_shuffle_down` and `simd_broadcast` on floats, the integer ones as + `.s` and `.u`, and `simd_any`, `simd_all`. There are no bfloat overloads. +- `convert` between float types, and `convert.s`, `convert.u` between floats + and integers. Apple's compiler uses LLVM casts for scalar f32 to and from + f16 or bf16, and between integers. +- `wg.barrier` and `simdgroup.barrier` (`threadgroup_barrier`, + `simdgroup_barrier`), with `#air.mem_flags`. +- `simdgroup_matrix_8x8_load`, `_store` and `_multiply_accumulate`. A + `simdgroup_matrix` is a `vector<64xT>`, and `thread_elements()` its + elements 0 and 1. + +Float operations carry `fastmath` flags, which become the call's flags (Apple's +compiler marks every float call `fast` under fast math). SIMD-group, barrier +and matrix operations have unknown effects, so they are not speculated or +moved across divergent control flow. Arithmetic, memory and control flow use +the arith, cf, scf, vector and llvm dialects, with pointers as `!llvm.ptr` in +AIR's address spaces: 0 thread, 1 device, 2 constant, 3 threadgroup (a buffer +is never in thread memory). + +## Kernels + +A kernel is a void `func.func` with `air.kernel`. Each argument carries one +`!air.kernel` descriptor: + +- `air.builtin = #air.builtin`: the position and + size builtins are `uint` or `ushort` scalars or 2- and 3-element vectors, + the index builtins (`thread_index_in_simdgroup`, ...) scalars. +- `air.buffer = #air.buffer<...>` on an `!llvm.ptr`: the `[[buffer(n)]]` or + `[[threadgroup(n)]]` index, the address space, the access (`const` pointers + are `read`), the pointee type with its Metal name, size and alignment, the + size of a constant reference, `noalias` for `__restrict__`, and the fields + of a constant struct (`air.struct_type_info`). Device and constant buffers + share indices; threadgroup buffers have their own. +- `air.arg_name`, optionally, the argument's name. + +`air.fast_math_enable` on the module records that it was compiled with fast +math (`air.compile.fast_math_enable`). + +## Evidence + +Names, signatures, argument descriptors and flags follow what Apple's Metal +compiler 32023.830 (the `@metal_toolchain` repository) emits for MSL probes and +for XLA's Metal kernels, with `metal -std=metal4.0 -S -emit-llvm` (fast math, +-O2, as XLA compiles them). Only the overloads it emits are accepted. + +## Build and test + +```sh +bazel build //mlir/dialects/air //mlir/dialects/air:capi +bazel test //mlir/dialects/air:test +``` + +The Zig library is `:air`, the C++ dialect `:dialect` and its C API `:capi`. +`registerDialect(registry)` registers AIR, which loads the LLVM dialect; +`kernel` builds a `func.func`, so callers also register `func`. Attributes are +built through the C API (`BuiltinAttr`, `MemFlagsAttr`, `StructFieldAttr`, +`BufferAttr`), kernel arguments through `builtinArgumentAttr` and +`bufferArgumentAttr`, and every operation has a builder named after it +(`fast_exp`, `max_u`, `wg_barrier`, `simdgroup_matrix_8x8_load`). diff --git a/mlir/dialects/air/air.zig b/mlir/dialects/air/air.zig new file mode 100644 index 00000000..2cdf2d25 --- /dev/null +++ b/mlir/dialects/air/air.zig @@ -0,0 +1,536 @@ +//! Metal AIR intrinsics and kernel ABI annotations. A builder emits the +//! `air.*` intrinsic it names, minus the type suffix its types select: +//! `fast_exp` on f32 is `@air.fast_exp.f32`, `max_u` on `vector<2xi32>` is +//! `@air.max.u.v2i32`. Arithmetic, memory and control flow use the arith, +//! llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address spaces. +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const c = @import("c"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("air"); +} + +pub const Builtin = enum(u32) { + thread_position_in_grid, + threadgroup_position_in_grid, + thread_position_in_threadgroup, + threads_per_threadgroup, + threadgroups_per_grid, + threads_per_grid, + thread_index_in_simdgroup, + simdgroup_index_in_threadgroup, + threads_per_simdgroup, + simdgroups_per_threadgroup, + thread_index_in_threadgroup, +}; + +pub const AddressSpace = enum(u32) { thread = 0, device = 1, constant = 2, threadgroup = 3 }; + +pub const Access = enum(u32) { read, read_write }; + +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const none: MemFlags = .{}; +}; + +/// `#air.builtin`: Kernel argument builtin +pub const BuiltinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuiltin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: Builtin }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBuiltinAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) Builtin { + return @fromBackingInt(@intCast(c.mlirMetalAIRBuiltinAttrGetValue(self.ptr()))); + } +}; + +/// `#air.mem_flags`: Memory a barrier orders +pub const MemFlagsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRMemFlags; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: MemFlags }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRMemFlagsAttrGet(ctx.ptr(), @bitCast(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) MemFlags { + return @bitCast(c.mlirMetalAIRMemFlagsAttrGetValue(self.ptr())); + } +}; + +/// `#air.struct_field`: Field of a constant buffer struct +pub const StructFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRStructField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + offset: u64, + size: u64, + /// The array length, zero for a non-array field. + count: u64 = 0, + typeName: []const u8, + name: []const u8, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRStructFieldAttrGet(ctx.ptr(), args.offset, args.size, args.count, mlir.stringRef(args.typeName), mlir.stringRef(args.name)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn offset(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetOffset(self.ptr()); + } + + pub fn size(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetSize(self.ptr()); + } + + pub fn count(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetCount(self.ptr()); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetTypeName(self.ptr())); + } + + pub fn name(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetName(self.ptr())); + } +}; + +/// `#air.buffer`: Kernel buffer argument +pub const BufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + locationIndex: u32, + addressSpace: AddressSpace, + access: Access, + elementType: *const mlir.Type, + typeName: []const u8, + typeSize: u64, + typeAlignment: u64, + /// The size of a constant reference; zero omits it. + bufferSize: u64 = 0, + noalias_: bool = false, + fields: []const *const StructFieldAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBufferAttrGet( + ctx.ptr(), + args.locationIndex, + @backingInt(args.addressSpace), + @backingInt(args.access), + args.elementType.ptr(), + mlir.stringRef(args.typeName), + args.typeSize, + args.typeAlignment, + args.bufferSize, + args.noalias_, + @intCast(args.fields.len), + @ptrCast(args.fields.ptr), + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn locationIndex(self: *const Self) u32 { + return c.mlirMetalAIRBufferAttrGetLocationIndex(self.ptr()); + } + + pub fn addressSpace(self: *const Self) AddressSpace { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAddressSpace(self.ptr()))); + } + + pub fn access(self: *const Self) Access { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAccess(self.ptr()))); + } + + pub fn elementType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirMetalAIRBufferAttrGetElementType(self.ptr()).ptr); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRBufferAttrGetTypeName(self.ptr())); + } + + pub fn typeSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeSize(self.ptr()); + } + + pub fn typeAlignment(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeAlignment(self.ptr()); + } + + pub fn bufferSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetBufferSize(self.ptr()); + } + + pub fn noalias_(self: *const Self) bool { + return c.mlirMetalAIRBufferAttrGetNoalias(self.ptr()); + } + + pub fn numFields(self: *const Self) usize { + return @intCast(c.mlirMetalAIRBufferAttrGetNumFields(self.ptr())); + } + + pub fn field(self: *const Self, i: usize) *const StructFieldAttr { + return @ptrCast(c.mlirMetalAIRBufferAttrGetField(self.ptr(), @intCast(i)).ptr); + } +}; + +pub const ArgumentOpts = struct { + /// `air.arg_name`, the argument's name in reflection. + name: ?[]const u8 = null, +}; + +/// The argument dictionary of a kernel argument bound to `builtin`. +pub fn builtinArgumentAttr(ctx: *mlir.Context, builtin: Builtin, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.builtin", (try BuiltinAttr.get(ctx, .{ .value = builtin })).attribute()), opts); +} + +/// The argument dictionary of a kernel buffer argument. +pub fn bufferArgumentAttr(ctx: *mlir.Context, args: BufferAttr.InitArgs, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.buffer", (try BufferAttr.get(ctx, args)).attribute()), opts); +} + +fn argumentAttr(ctx: *mlir.Context, annotation: mlir.NamedAttribute, opts: ArgumentOpts) *const mlir.Attribute { + if (opts.name) |name| return .dict(ctx, &.{ annotation, .named(ctx, "air.arg_name", .string(ctx, name)) }); + return .dict(ctx, &.{annotation}); +} + +/// Marks `module` as compiled with fast math (`air.compile.fast_math_enable`). +pub fn enableFastMath(ctx: *mlir.Context, module: *mlir.Operation) void { + module.setAttributeByName("air.fast_math_enable", .unit(ctx)); +} + +pub const KernelArgs = struct { + name: []const u8, + block: *mlir.Block, + args_attributes: []const *const mlir.Attribute, + location: *const mlir.Location, + verify: bool = true, +}; + +/// A `func.func` with `air.kernel`, one argument dictionary per block +/// argument. Takes ownership of `block`; set `verify = false` while the body +/// is incomplete. +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + std.debug.assert(args.args_attributes.len == args.block.numArguments()); + return func.func(ctx, .{ + .name = args.name, + .block = args.block, + .args_attributes = args.args_attributes, + .results = &.{}, + .extra_attributes = &.{.named(ctx, "air.kernel", .unit(ctx))}, + .location = args.location, + .verify = args.verify, + }); +} + +pub const FloatOpts = struct { + fastmath: arith.FastMathFlags = .none, +}; + +/// The `fastmath` attribute of a float op, stored in the caller's `buf`. +fn fastmathAttrs(buf: *[1]mlir.NamedAttribute, ctx: *mlir.Context, fastmath: arith.FastMathFlags) []const mlir.NamedAttribute { + if (fastmath == arith.FastMathFlags.none) return &.{}; + const attr = arith.FastMathAttr.get(ctx, .{ .value = fastmath }) catch unreachable; + buf[0] = .named(ctx, "fastmath", attr.attribute()); + return buf; +} + +fn floatUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn floatShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .location = location, + }); + } + }; +} + +fn convertOp(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn barrier(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, mem_flags: MemFlags, location: *const mlir.Location) *mlir.Operation { + const flags = MemFlagsAttr.get(ctx, .{ .value = mem_flags }) catch unreachable; + return mlir.Operation.make(ctx, name, .{ + .attributes = &.{.named(ctx, "mem_flags", flags.attribute())}, + .location = location, + }); + } + }; +} + +pub const exp = floatUnary("air.exp").call; +pub const fast_exp = floatUnary("air.fast_exp").call; +pub const exp2 = floatUnary("air.exp2").call; +pub const fast_exp2 = floatUnary("air.fast_exp2").call; +pub const log = floatUnary("air.log").call; +pub const fast_log = floatUnary("air.fast_log").call; +pub const log2 = floatUnary("air.log2").call; +pub const fast_log2 = floatUnary("air.fast_log2").call; +pub const tanh = floatUnary("air.tanh").call; +pub const fast_tanh = floatUnary("air.fast_tanh").call; +pub const sqrt = floatUnary("air.sqrt").call; +pub const fast_sqrt = floatUnary("air.fast_sqrt").call; +pub const rsqrt = floatUnary("air.rsqrt").call; +pub const fast_rsqrt = floatUnary("air.fast_rsqrt").call; +pub const fabs = floatUnary("air.fabs").call; +pub const fast_fabs = floatUnary("air.fast_fabs").call; +pub const ceil = floatUnary("air.ceil").call; +pub const fast_ceil = floatUnary("air.fast_ceil").call; +pub const floor = floatUnary("air.floor").call; +pub const fast_floor = floatUnary("air.fast_floor").call; +pub const round = floatUnary("air.round").call; +pub const fast_round = floatUnary("air.fast_round").call; +pub const trunc = floatUnary("air.trunc").call; +pub const fast_trunc = floatUnary("air.fast_trunc").call; +pub const sin = floatUnary("air.sin").call; +pub const fast_sin = floatUnary("air.fast_sin").call; +pub const cos = floatUnary("air.cos").call; +pub const fast_cos = floatUnary("air.fast_cos").call; +pub const fmax = floatBinary("air.fmax").call; +pub const fast_fmax = floatBinary("air.fast_fmax").call; +pub const fmin = floatBinary("air.fmin").call; +pub const fast_fmin = floatBinary("air.fast_fmin").call; +pub const pow = floatBinary("air.pow").call; +pub const fast_pow = floatBinary("air.fast_pow").call; +pub const fmod = floatBinary("air.fmod").call; +pub const fast_fmod = floatBinary("air.fast_fmod").call; +pub const fma = floatTernary("air.fma").call; +pub const clamp = floatTernary("air.clamp").call; +pub const fast_clamp = floatTernary("air.fast_clamp").call; +pub const abs_s = intUnary("air.abs.s").call; +pub const max_s = intBinary("air.max.s").call; +pub const max_u = intBinary("air.max.u").call; +pub const min_s = intBinary("air.min.s").call; +pub const min_u = intBinary("air.min.u").call; +pub const clamp_s = intTernary("air.clamp.s").call; +pub const clamp_u = intTernary("air.clamp.u").call; +pub const simd_sum = floatUnary("air.simd_sum").call; +pub const simd_sum_s = intUnary("air.simd_sum.s").call; +pub const simd_sum_u = intUnary("air.simd_sum.u").call; +pub const simd_max = floatUnary("air.simd_max").call; +pub const simd_max_s = intUnary("air.simd_max.s").call; +pub const simd_max_u = intUnary("air.simd_max.u").call; +pub const simd_min = floatUnary("air.simd_min").call; +pub const simd_min_s = intUnary("air.simd_min.s").call; +pub const simd_min_u = intUnary("air.simd_min.u").call; +pub const simd_prefix_exclusive_sum = floatUnary("air.simd_prefix_exclusive_sum").call; +pub const simd_shuffle = floatShuffle("air.simd_shuffle").call; +pub const simd_shuffle_s = intShuffle("air.simd_shuffle.s").call; +pub const simd_shuffle_u = intShuffle("air.simd_shuffle.u").call; +pub const simd_shuffle_xor = floatShuffle("air.simd_shuffle_xor").call; +pub const simd_shuffle_xor_s = intShuffle("air.simd_shuffle_xor.s").call; +pub const simd_shuffle_xor_u = intShuffle("air.simd_shuffle_xor.u").call; +pub const simd_shuffle_down = floatShuffle("air.simd_shuffle_down").call; +pub const simd_shuffle_down_s = intShuffle("air.simd_shuffle_down.s").call; +pub const simd_shuffle_down_u = intShuffle("air.simd_shuffle_down.u").call; +pub const simd_broadcast = floatShuffle("air.simd_broadcast").call; +pub const simd_broadcast_s = intShuffle("air.simd_broadcast.s").call; +pub const simd_broadcast_u = intShuffle("air.simd_broadcast.u").call; +pub const simd_any = intUnary("air.simd_any").call; +pub const simd_all = intUnary("air.simd_all").call; +pub const convert = convertOp("air.convert").call; +pub const convert_s = convertOp("air.convert.s").call; +pub const convert_u = convertOp("air.convert.u").call; +pub const wg_barrier = barrier("air.wg.barrier").call; +pub const simdgroup_barrier = barrier("air.simdgroup.barrier").call; + +pub fn dot(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.dot", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_().isA(mlir.ShapedType).?.elementType()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +/// `simdgroup_load`: `extents`, `strides` and `origin` are `vector<2xi64>`. +pub fn simdgroup_matrix_8x8_load(ctx: *mlir.Context, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_load", .{ + .operands = .{ .flat = &.{ pointer, extents, strides, origin } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +pub fn simdgroup_matrix_8x8_store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_store", .{ + .operands = .{ .flat = &.{ value, pointer, extents, strides, origin } }, + .location = location, + }); +} + +/// `simdgroup_multiply_accumulate`: `a * b + c`. +pub fn simdgroup_matrix_8x8_multiply_accumulate(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_multiply_accumulate", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} diff --git a/mlir/dialects/air/capi.cc b/mlir/dialects/air/capi.cc new file mode 100644 index 00000000..64df5d20 --- /dev/null +++ b/mlir/dialects/air/capi.cc @@ -0,0 +1,189 @@ +//===- capi.cc - C API for the Metal AIR dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/SmallVector.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(MetalAIR, air, AIRDialect) + +static MLIRContext *loadAIR(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +static MlirAttribute invalid(MLIRContext *ctx, const Twine &message) { + emitError(UnknownLoc::get(ctx), message); + return {nullptr}; +} + +// Attributes + +bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional builtin = symbolizeBuiltin(value); + if (!builtin) + return invalid(ctx, "unknown AIR builtin " + Twine(value)); + return wrap(BuiltinAttr::get(ctx, *builtin)); +} + +uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional flags = symbolizeMemFlags(value); + if (!flags) + return invalid(ctx, "unknown AIR memory flags " + Twine(value)); + return wrap(MemFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRStructFieldAttrGet(MlirContext context, + uint64_t offset, uint64_t size, + uint64_t count, + MlirStringRef typeName, + MlirStringRef name) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + return wrap(StructFieldAttr::getChecked(UnknownLoc::get(ctx), ctx, offset, + size, count, unwrap(typeName), + unwrap(name))); +} + +uint64_t mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr) { + return cast(unwrap(attr)).getOffset(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr) { + return cast(unwrap(attr)).getSize(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr) { + return cast(unwrap(attr)).getCount(); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getName()); +} + +bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional space = symbolizeAddressSpace(addressSpace); + std::optional mode = symbolizeAccess(access); + if (!space || !mode) + return invalid(ctx, "unknown AIR buffer address space or access"); + Type element = unwrap(elementType); + if (!element || element.getContext() != ctx) + return invalid(ctx, + "buffer element type must belong to the attribute context"); + if (numFields < 0 || (numFields > 0 && !fields)) + return invalid(ctx, "invalid AIR struct fields"); + SmallVector structFields; + for (intptr_t i = 0; i < numFields; ++i) { + auto field = dyn_cast_or_null(unwrap(fields[i])); + if (!field || field.getContext() != ctx) + return invalid(ctx, "AIR struct fields must be #air.struct_field of " + "the attribute context"); + structFields.push_back(field); + } + return wrap(BufferAttr::getChecked( + UnknownLoc::get(ctx), ctx, locationIndex, *space, *mode, element, + unwrap(typeName), typeSize, typeAlignment, bufferSize, noalias, + ArrayRef(structFields))); +} + +uint32_t mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr) { + return cast(unwrap(attr)).getLocationIndex(); +} + +uint32_t mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr) { + return static_cast( + cast(unwrap(attr)).getAddressSpace()); +} + +uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getAccess()); +} + +MlirType mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getElementType()); +} + +MlirStringRef mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeSize(); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeAlignment(); +} + +uint64_t mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr) { + return cast(unwrap(attr)).getBufferSize(); +} + +bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr) { + return cast(unwrap(attr)).getNoalias(); +} + +intptr_t mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr) { + return cast(unwrap(attr)).getFields().size(); +} + +MlirAttribute mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos) { + return wrap(cast(unwrap(attr)).getFields()[pos]); +} diff --git a/mlir/dialects/air/capi.h b/mlir/dialects/air/capi.h new file mode 100644 index 00000000..f91b9f64 --- /dev/null +++ b/mlir/dialects/air/capi.h @@ -0,0 +1,81 @@ +//===- capi.h - C API for the Metal AIR dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AIR_CAPI_H +#define MLIR_DIALECTS_AIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(MetalAIR, air); + +// Attributes. Invalid parameters emit a diagnostic and return a null handle; +// typed getters require a handle of their attribute kind. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr); +/// `value` is a `mlir::metal::air::Builtin`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr); +/// `value` is a `mlir::metal::air::MemFlags` bit set. +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRStructFieldAttrGet( + MlirContext context, uint64_t offset, uint64_t size, uint64_t count, + MlirStringRef typeName, MlirStringRef name); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr); +/// `addressSpace` is a `mlir::metal::air::AddressSpace` and `access` a +/// `mlir::metal::air::Access`. Sizes are bytes; a zero `bufferSize` omits it. +/// `elementType` and `fields` must belong to `context`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AIR_CAPI_H diff --git a/mlir/dialects/air/test.zig b/mlir/dialects/air/test.zig new file mode 100644 index 00000000..57f0427e --- /dev/null +++ b/mlir/dialects/air/test.zig @@ -0,0 +1,439 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "air.abs.s", + "air.ceil", + "air.clamp", + "air.clamp.s", + "air.clamp.u", + "air.convert", + "air.convert.s", + "air.convert.u", + "air.cos", + "air.dot", + "air.exp", + "air.exp2", + "air.fabs", + "air.fast_ceil", + "air.fast_clamp", + "air.fast_cos", + "air.fast_exp", + "air.fast_exp2", + "air.fast_fabs", + "air.fast_floor", + "air.fast_fmax", + "air.fast_fmin", + "air.fast_fmod", + "air.fast_log", + "air.fast_log2", + "air.fast_pow", + "air.fast_round", + "air.fast_rsqrt", + "air.fast_sin", + "air.fast_sqrt", + "air.fast_tanh", + "air.fast_trunc", + "air.floor", + "air.fma", + "air.fmax", + "air.fmin", + "air.fmod", + "air.log", + "air.log2", + "air.max.s", + "air.max.u", + "air.min.s", + "air.min.u", + "air.pow", + "air.round", + "air.rsqrt", + "air.simd_all", + "air.simd_any", + "air.simd_broadcast", + "air.simd_broadcast.s", + "air.simd_broadcast.u", + "air.simd_max", + "air.simd_max.s", + "air.simd_max.u", + "air.simd_min", + "air.simd_min.s", + "air.simd_min.u", + "air.simd_prefix_exclusive_sum", + "air.simd_shuffle", + "air.simd_shuffle.s", + "air.simd_shuffle.u", + "air.simd_shuffle_down", + "air.simd_shuffle_down.s", + "air.simd_shuffle_down.u", + "air.simd_shuffle_xor", + "air.simd_shuffle_xor.s", + "air.simd_shuffle_xor.u", + "air.simd_sum", + "air.simd_sum.s", + "air.simd_sum.u", + "air.simdgroup.barrier", + "air.simdgroup_matrix_8x8_load", + "air.simdgroup_matrix_8x8_multiply_accumulate", + "air.simdgroup_matrix_8x8_store", + "air.sin", + "air.sqrt", + "air.tanh", + "air.trunc", + "air.wg.barrier", +}; + +test { + std.testing.refAllDecls(air); + inline for (.{ air.BuiltinAttr, air.MemFlagsAttr, air.StructFieldAttr, air.BufferAttr }) |T| std.testing.refAllDecls(T); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ air, arith, func, llvm }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, value: anytype) !void { + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{value}); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(value.attribute())); +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "AIR attributes use the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (std.meta.tags(air.Builtin)) |builtin| { + const attr = try air.BuiltinAttr.get(ctx, .{ .value = builtin }); + try std.testing.expectEqual(builtin, attr.value()); + try expectRoundTrip(ctx, attr); + } + for ([_]air.MemFlags{ .none, .{ .device = true }, .{ .threadgroup = true }, .{ .device = true, .threadgroup = true, .texture = true } }) |flags| { + const attr = try air.MemFlagsAttr.get(ctx, .{ .value = flags }); + try std.testing.expectEqual(flags, attr.value()); + try expectRoundTrip(ctx, attr); + } + try std.testing.expectError(error.InvalidMlir, air.MemFlagsAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); + + const field = try air.StructFieldAttr.get(ctx, .{ .offset = 4, .size = 4, .count = 3, .typeName = "float", .name = "s" }); + try std.testing.expectEqual(4, field.offset()); + try std.testing.expectEqual(4, field.size()); + try std.testing.expectEqual(3, field.count()); + try std.testing.expectEqualStrings("float", field.typeName()); + try std.testing.expectEqualStrings("s", field.name()); + try expectRoundTrip(ctx, field); + try std.testing.expectError(error.InvalidMlir, air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 0, .typeName = "int", .name = "n" })); + + const i32_type: *const mlir.Type = .int(ctx, .i32); + const f32_type: *const mlir.Type = .float(ctx, .f32); + const device: air.BufferAttr.InitArgs = .{ + .locationIndex = 1, + .addressSpace = .device, + .access = .read, + .elementType = .vector(&.{4}, .float(ctx, .bf16)), + .typeName = "bfloat4", + .typeSize = 8, + .typeAlignment = 8, + .noalias_ = true, + }; + const buffer = try air.BufferAttr.get(ctx, device); + try std.testing.expectEqual(1, buffer.locationIndex()); + try std.testing.expectEqual(air.AddressSpace.device, buffer.addressSpace()); + try std.testing.expectEqual(air.Access.read, buffer.access()); + try std.testing.expect(buffer.elementType().eql(device.elementType)); + try std.testing.expectEqualStrings("bfloat4", buffer.typeName()); + try std.testing.expectEqual(8, buffer.typeSize()); + try std.testing.expectEqual(8, buffer.typeAlignment()); + try std.testing.expectEqual(0, buffer.bufferSize()); + try std.testing.expect(buffer.noalias_()); + try std.testing.expectEqual(0, buffer.numFields()); + try expectRoundTrip(ctx, buffer); + + // `struct A { int n; float s[3]; }`, as Apple's compiler describes it. + const fields = [_]*const air.StructFieldAttr{ + try air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 4, .typeName = "int", .name = "n" }), + field, + }; + const constant: air.BufferAttr.InitArgs = .{ + .locationIndex = 0, + .addressSpace = .constant, + .access = .read, + .elementType = llvm.structType(ctx, &.{ i32_type, llvm.arrayType(f32_type, 3) }), + .typeName = "A", + .typeSize = 16, + .typeAlignment = 4, + .bufferSize = 16, + .fields = &fields, + }; + const structured = try air.BufferAttr.get(ctx, constant); + try std.testing.expectEqual(2, structured.numFields()); + try std.testing.expect(structured.field(1).eql(field)); + try expectRoundTrip(ctx, structured); + + var invalid = device; + invalid.typeSize = 16; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .constant; + invalid.access = .read_write; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .thread; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .threadgroup; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.bufferSize = 8; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.addressSpace = .device; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + + const other = try testContext(); + defer other.deinit(); + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(other, device)); +} + +test "all AIR operations construct and verify" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32x4: *const mlir.Type = .vector(&.{4}, .float(ctx, .f32)); + const i32x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i32)); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init( + &.{ .float(ctx, .f32), .float(ctx, .f16), f32x4, .int(ctx, .i32), i32x2, .int(ctx, .i16), .int(ctx, .i1), llvm.pointerType(ctx, air.AddressSpace.threadgroup), i64x2, .vector(&.{64}, .float(ctx, .f16)), .vector(&.{64}, .float(ctx, .f32)), .vector(&.{4}, .float(ctx, .bf16)) }, + &.{ loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc }, + ); + defer block.deinit(); + const f = block.argument(0); + const h = block.argument(1); + const v = block.argument(2); + const i = block.argument(3); + const iv = block.argument(4); + const lane = block.argument(5); + const b = block.argument(6); + const fast: air.FloatOpts = .{ .fastmath = .fast }; + var ops: std.ArrayList(*mlir.Operation) = .empty; + defer ops.deinit(std.testing.allocator); + inline for (.{ air.exp, air.exp2, air.log, air.log2, air.tanh, air.sqrt, air.rsqrt, air.fabs, air.ceil, air.floor, air.round, air.trunc, air.sin, air.cos, air.simd_sum, air.simd_max, air.simd_min, air.simd_prefix_exclusive_sum }) |op| { + try ops.append(std.testing.allocator, op(ctx, h, .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fast_exp, air.fast_exp2, air.fast_log, air.fast_log2, air.fast_tanh, air.fast_sqrt, air.fast_rsqrt, air.fast_fabs, air.fast_ceil, air.fast_floor, air.fast_round, air.fast_trunc, air.fast_sin, air.fast_cos }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fmax, air.fmin, air.pow, air.fmod, air.fast_fmax, air.fast_fmin, air.fast_pow, air.fast_fmod }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, f, fast, loc).appendTo(block)); + } + inline for (.{ air.fma, air.clamp, air.fast_clamp }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, v, v, fast, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.dot(ctx, v, v, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.abs_s(ctx, i, loc).appendTo(block)); + inline for (.{ air.max_s, air.max_u, air.min_s, air.min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, iv, loc).appendTo(block)); + } + inline for (.{ air.clamp_s, air.clamp_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, i, i, loc).appendTo(block)); + } + inline for (.{ air.simd_sum_s, air.simd_sum_u, air.simd_max_s, air.simd_max_u, air.simd_min_s, air.simd_min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle, air.simd_shuffle_xor, air.simd_shuffle_down, air.simd_broadcast }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, lane, fast, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle_s, air.simd_shuffle_u, air.simd_shuffle_xor_s, air.simd_shuffle_xor_u, air.simd_shuffle_down_s, air.simd_shuffle_down_u, air.simd_broadcast_s, air.simd_broadcast_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_any, air.simd_all }) |op| { + try ops.append(std.testing.allocator, op(ctx, b, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.convert(ctx, block.argument(11), f32x4, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert(ctx, h, .float(ctx, .bf16), .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_s(ctx, i, .float(ctx, .f32), fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_u(ctx, f, .int(ctx, .i32), .{}, loc).appendTo(block)); + inline for (.{ air.wg_barrier, air.simdgroup_barrier }) |op| { + try ops.append(std.testing.allocator, op(ctx, .{ .threadgroup = true }, loc).appendTo(block)); + } + const shape = block.argument(8); + const matrix = air.simdgroup_matrix_8x8_load(ctx, block.argument(7), shape, shape, shape, .vector(&.{64}, .float(ctx, .f16)), fast, loc).appendTo(block); + const product = air.simdgroup_matrix_8x8_multiply_accumulate(ctx, matrix.result(0), block.argument(9), block.argument(10), .vector(&.{64}, .float(ctx, .f32)), fast, loc).appendTo(block); + const stored = air.simdgroup_matrix_8x8_store(ctx, product.result(0), block.argument(7), shape, shape, shape, loc).appendTo(block); + try ops.appendSlice(std.testing.allocator, &.{ matrix, product, stored }); + for (ops.items) |op| try std.testing.expect(op.verify()); +} + +test "AIR operations round trip through their custom assembly" { + const ctx = try testContext(); + defer ctx.deinit(); + const source = + \\func.func @ops(%f: f32, %v: vector<4xbf16>, %i: i32, %s: i16, %b: i1, %p: !llvm.ptr<3>, %d: vector<2xi64>, %m: vector<64xf32>) { + \\ %0 = air.fast_exp %f fastmath : f32 + \\ %1 = air.tanh %f : f32 + \\ %2 = air.fma %f, %f, %f fastmath : f32 + \\ %3 = air.max.s %i, %i : i32 + \\ %4 = air.simd_shuffle_xor %f, %s fastmath : f32 + \\ %5 = air.simd_sum.u %i : i32 + \\ %6 = air.simd_any %b + \\ %7 = air.convert %v fastmath : vector<4xbf16> to vector<4xf32> + \\ %8 = air.convert.s %i fastmath : i32 to f32 + \\ %9 = air.dot %7, %7 fastmath : vector<4xf32> -> f32 + \\ air.wg.barrier + \\ air.simdgroup.barrier + \\ %10 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d fastmath : !llvm.ptr<3> -> vector<64xf32> + \\ %11 = air.simdgroup_matrix_8x8_multiply_accumulate %10, %m, %m fastmath : vector<64xf32>, vector<64xf32>, vector<64xf32> -> vector<64xf32> + \\ air.simdgroup_matrix_8x8_store %11, %p, %d, %d, %d : vector<64xf32>, !llvm.ptr<3> + \\ return + \\} + ; + const module = try mlir.Module.parse(ctx, source); + defer module.deinit(); + var first: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer first.deinit(); + try first.writer.print("{f}", .{module.operation()}); + try std.testing.expect(std.mem.find(u8, first.written(), "\"air.") == null); + const reparsed = try mlir.Module.parse(ctx, first.written()); + defer reparsed.deinit(); + var second: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer second.deinit(); + try second.writer.print("{f}", .{reparsed.operation()}); + try std.testing.expectEqualStrings(first.written(), second.written()); +} + +test "AIR verifier rejects types Apple's compiler has no intrinsic for" { + const ctx = try testContext(); + defer ctx.deinit(); + const rejected = [_][]const u8{ + "func.func @f(%x: f16) { %0 = air.fast_exp %x : f16\n return }", + "func.func @f(%x: bf16) { %0 = air.exp %x : bf16\n return }", + "func.func @f(%x: f32) { %0 = air.max.s %x, %x : f32\n return }", + "func.func @f(%x: i64) { %0 = air.simd_sum.s %x : i64\n return }", + "func.func @f(%x: vector<8xf32>) { %0 = air.simd_sum %x : vector<8xf32>\n return }", + "func.func @f(%x: f32) { %0 = air.convert %x : f32 to f32\n return }", + "func.func @f(%x: i32) { %0 = air.convert %x : i32 to f32\n return }", + "func.func @f(%x: f32) { %0 = air.convert.s %x : f32 to f16\n return }", + "func.func @f(%x: f32) { %0 = air.convert.u %x fastmath : f32 to i32\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.convert %x : vector<4xf32> to vector<2xf16>\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.dot %x, %x : vector<4xf32> -> f16\n return }", + "func.func @f(%p: !llvm.ptr<2>, %d: vector<2xi64>) { %0 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d : !llvm.ptr<2> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xbf16>, %b: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %b, %c : vector<64xbf16>, vector<64xf16>, vector<64xf32> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %a, %c : vector<64xf16>, vector<64xf16>, vector<64xf32> -> vector<64xf16>\n return }", + }; + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR verifier takes the 8-bit integers Apple's compiler emits for char" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = try mlir.Module.parse(ctx, + \\func.func @f(%c: i8, %v: vector<2xi8>, %f: f32, %w: vector<2xf32>, %l: i16) { + \\ %0 = air.convert.s %c fastmath : i8 to f32 + \\ %1 = air.convert.u %f : f32 to i8 + \\ %2 = air.convert.s %v fastmath : vector<2xi8> to vector<2xf32> + \\ %3 = air.convert.s %w : vector<2xf32> to vector<2xi8> + \\ %4 = air.max.s %c, %c : i8 + \\ %5 = air.min.u %v, %v : vector<2xi8> + \\ %6 = air.abs.s %c : i8 + \\ %7 = air.clamp.u %c, %c, %c : i8 + \\ %8 = air.simd_shuffle.s %c, %l : i8 + \\ %9 = air.simd_sum.s %c : i8 + \\ %10 = air.simd_max.u %c : i8 + \\ return + \\} + ); + module.deinit(); +} + +test "AIR kernel ABI" { + const ctx = try testContext(); + defer ctx.deinit(); + const device = "#air.buffer"; + const threadgroup = "#air.buffer"; + const accepted = [_][]const u8{ + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.arg_name = \"a\"}, %b: !llvm.ptr<3> {air.buffer = " ++ threadgroup ++ "}, %t: vector<3xi32> {air.builtin = #air.builtin}, %l: i16 {air.builtin = #air.builtin}, %s: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "module attributes {air.fast_math_enable} {}", + }; + const rejected = [_][]const u8{ + "func.func @k(%a: i32) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<3> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ "}, %b: !llvm.ptr<1> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<3xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<4xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i64 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin}, %b: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin, air.arg_name = \"\"}) attributes {air.kernel} { return }", + "func.func @k() -> i32 attributes {air.kernel} { %c = arith.constant 0 : i32\n return %c : i32 }", + "func.func @h(%a: i32 {air.builtin = #air.builtin}) { return }", + "func.func @h() attributes {air.fast_math_enable} { return }", + "func.func @h() attributes {air.unknown} { return }", + }; + for (accepted) |source| { + const module = try mlir.Module.parse(ctx, source); + module.deinit(); + } + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR kernel builder" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = mlir.Block.init(&.{ llvm.pointerType(ctx, air.AddressSpace.device), .vector(&.{3}, .int(ctx, .i32)) }, &.{ loc, loc }); + _ = air.wg_barrier(ctx, .{ .threadgroup = true }, loc).appendTo(block); + _ = func.returns(ctx, &.{}, loc).appendTo(block); + _ = air.kernel(ctx, .{ + .name = "example", + .block = block, + .location = loc, + .args_attributes = &.{ + try air.bufferArgumentAttr(ctx, .{ + .locationIndex = 0, + .addressSpace = .device, + .access = .read_write, + .elementType = .float(ctx, .f32), + .typeName = "float", + .typeSize = 4, + .typeAlignment = 4, + }, .{ .name = "out" }), + try air.builtinArgumentAttr(ctx, .thread_position_in_grid, .{}), + }, + }).appendTo(module.body()); + air.enableFastMath(ctx, module.operation()); + try std.testing.expect(module.operation().verify()); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{module.operation()}); + const parsed = try mlir.Module.parse(ctx, writer.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + try std.testing.expect(std.mem.find(u8, writer.written(), "air.arg_name = \"out\"") != null); +} diff --git a/mlir/dialects/arith/BUILD.bazel b/mlir/dialects/arith/BUILD.bazel new file mode 100644 index 00000000..90c84ff0 --- /dev/null +++ b/mlir/dialects/arith/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "arith", + import_name = "mlir/dialects/arith", + main = "arith.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIArith", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":arith", + "//mlir", + ], +) diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig new file mode 100644 index 00000000..c45ce5b5 --- /dev/null +++ b/mlir/dialects/arith/arith.zig @@ -0,0 +1,495 @@ +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("arith"); +} + +// ============================================================================= +// Predicate enums (match ArithBase.td exactly) +// ============================================================================= + +/// Arith cmpi predicates. Values match mlir::arith::CmpIPredicate (I64EnumAttr). +pub const CmpIPredicate = enum(i64) { + eq = 0, + ne = 1, + slt = 2, + sle = 3, + sgt = 4, + sge = 5, + ult = 6, + ule = 7, + ugt = 8, + uge = 9, +}; + +/// Arith cmpf predicates. Values match mlir::arith::CmpFPredicate. +pub const CmpFPredicate = enum(i64) { + false = 0, + oeq = 1, + ogt = 2, + oge = 3, + olt = 4, + ole = 5, + one = 6, + ord = 7, + ueq = 8, + ugt = 9, + uge = 10, + ult = 11, + ule = 12, + une = 13, + uno = 14, + true = 15, +}; + +// ============================================================================= +// Flag attributes +// ============================================================================= + +/// `mlir::arith::FastMathFlags`, one bit per flag. +pub const FastMathFlags = packed struct(u32) { + reassoc: bool = false, + nnan: bool = false, + ninf: bool = false, + nsz: bool = false, + arcp: bool = false, + contract: bool = false, + afn: bool = false, + _padding: u25 = 0, + + pub const none: FastMathFlags = .{}; + pub const fast: FastMathFlags = .{ .reassoc = true, .nnan = true, .ninf = true, .nsz = true, .arcp = true, .contract = true, .afn = true }; +}; + +/// `#arith.fastmath<...>`. +pub const FastMathAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithFastMath; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "fastmath"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FastMathFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithFastMathAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) FastMathFlags { + return @bitCast(c.mlirArithFastMathAttrGetValue(self.ptr())); + } +}; + +/// `fastmath = #arith.fastmath`: the fast-math flags of a float +/// operation. +pub fn fastmath(ctx: *mlir.Context, flags: FastMathFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "fastmath", (try FastMathAttr.get(ctx, .{ .value = flags })).attribute()); +} + +/// `mlir::arith::IntegerOverflowFlags`, one bit per flag. +pub const IntegerOverflowFlags = packed struct(u32) { + nsw: bool = false, + nuw: bool = false, + _padding: u30 = 0, + + pub const none: IntegerOverflowFlags = .{}; +}; + +/// `#arith.overflow<...>`. +pub const IntegerOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithIntegerOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerOverflowFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithIntegerOverflowAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IntegerOverflowFlags { + return @bitCast(c.mlirArithIntegerOverflowAttrGetValue(self.ptr())); + } +}; + +/// `overflowFlags = #arith.overflow`: the overflow flags of an +/// integer operation. +pub fn overflowFlags(ctx: *mlir.Context, flags: IntegerOverflowFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "overflowFlags", (try IntegerOverflowAttr.get(ctx, .{ .value = flags })).attribute()); +} + +// ============================================================================= +// Constants +// ============================================================================= + +/// arith.constant for an integer scalar. `value` is reinterpreted to `int_type`'s bit width. +pub fn constant_int( + ctx: *mlir.Context, + value: i64, + int_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{int_type} }, + .attributes = &.{.named(ctx, "value", .intFromType(@ptrCast(int_type), value))}, + .location = location, + }); +} + +/// arith.constant for a float scalar of the given FloatTypes kind. +pub fn constant_float( + ctx: *mlir.Context, + value: f64, + comptime ft: mlir.FloatTypes, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{.float(ctx, ft)} }, + .attributes = &.{.named(ctx, "value", .float(ctx, ft, value))}, + .location = location, + }); +} + +/// arith.constant for an `index`-typed integer. +pub fn constant_index(ctx: *mlir.Context, value: i64, location: *const mlir.Location) *mlir.Operation { + const idx_ty = mlir.Type.index(ctx); + const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(idx_ty.ptr(), value).ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{idx_ty} }, + .attributes = &.{.named(ctx, "value", attr)}, + .location = location, + }); +} + +// ============================================================================= +// Binary ops (generated from shared template) +// ============================================================================= + +fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Integer arithmetic +pub const addi = binaryOp("arith.addi").call; +pub const subi = binaryOp("arith.subi").call; +pub const muli = binaryOp("arith.muli").call; +pub const divsi = binaryOp("arith.divsi").call; +pub const divui = binaryOp("arith.divui").call; +pub const remsi = binaryOp("arith.remsi").call; +pub const remui = binaryOp("arith.remui").call; + +// Integer bitwise / shifts +pub const andi = binaryOp("arith.andi").call; +pub const ori = binaryOp("arith.ori").call; +pub const xori = binaryOp("arith.xori").call; +pub const shli = binaryOp("arith.shli").call; +pub const shrsi = binaryOp("arith.shrsi").call; +pub const shrui = binaryOp("arith.shrui").call; + +// Integer min/max +pub const maxsi = binaryOp("arith.maxsi").call; +pub const maxui = binaryOp("arith.maxui").call; +pub const minsi = binaryOp("arith.minsi").call; +pub const minui = binaryOp("arith.minui").call; + +// Ceil/floor div +pub const ceildivsi = binaryOp("arith.ceildivsi").call; +pub const ceildivui = binaryOp("arith.ceildivui").call; +pub const floordivsi = binaryOp("arith.floordivsi").call; + +// Float arithmetic +pub const addf = binaryOp("arith.addf").call; +pub const subf = binaryOp("arith.subf").call; +pub const mulf = binaryOp("arith.mulf").call; +pub const divf = binaryOp("arith.divf").call; +pub const remf = binaryOp("arith.remf").call; + +// Float min/max +pub const maximumf = binaryOp("arith.maximumf").call; +pub const maxnumf = binaryOp("arith.maxnumf").call; +pub const minimumf = binaryOp("arith.minimumf").call; +pub const minnumf = binaryOp("arith.minnumf").call; + +// ============================================================================= +// Unary ops +// ============================================================================= + +pub fn negf(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.negf", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.flush_denormals — clamp subnormals to zero (SameOperandsAndResultType). +pub fn flush_denormals(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.flush_denormals", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.constant for a dense splat: builds `arith.constant dense : T` +/// where `T` is a ranked tensor type. Useful for tensor-typed zero/one constants. +/// For float splats, use `constant_dense_splat_f`; for ints, `constant_dense_splat_i`. +pub fn constant_dense_splat_f( + ctx: *mlir.Context, + value: f64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +pub fn constant_dense_splat_i( + ctx: *mlir.Context, + value: i64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirIntegerAttrGet(elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +// ============================================================================= +// Extended-result binary ops — return two values +// ============================================================================= + +/// arith.addui_extended — unsigned add, returns (sum, overflow: i1 / i1-tensor). +/// The overflow type is typically i1 (scalar) or tensor<...xi1> matching lhs shape. +pub fn addui_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + overflow_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.addui_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), overflow_type } }, + .location = location, + }); +} + +/// arith.mulsi_extended — signed multiply, returns (low, high) both same type as lhs. +pub fn mulsi_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mulsi_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +/// arith.mului_extended — unsigned multiply, returns (low, high) both same type as lhs. +pub fn mului_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mului_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// Cast ops (1 operand, explicit result type) +// ============================================================================= + +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const extsi = castOp("arith.extsi").call; +pub const extui = castOp("arith.extui").call; +pub const extf = castOp("arith.extf").call; +pub const trunci = castOp("arith.trunci").call; +pub const truncf = castOp("arith.truncf").call; +pub const sitofp = castOp("arith.sitofp").call; +pub const uitofp = castOp("arith.uitofp").call; +pub const fptosi = castOp("arith.fptosi").call; +pub const fptoui = castOp("arith.fptoui").call; +pub const bitcast = castOp("arith.bitcast").call; +pub const index_cast = castOp("arith.index_cast").call; +pub const index_castui = castOp("arith.index_castui").call; + +// ============================================================================= +// convertf — same-bitwidth float cast (e.g. f16↔bf16). Not expressible via +// arith.extf or arith.truncf. +// ============================================================================= + +/// Matches `Arith_RoundingModeAttr` (I32EnumAttr; see arith_dialect.td). +pub const RoundingMode = enum(i32) { + to_nearest_even = 0, + downward = 1, + upward = 2, + toward_zero = 3, + to_nearest_away = 4, +}; + +pub const ConvertFOpts = struct { + rounding: ?RoundingMode = null, +}; + +pub fn convertf( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + } + return mlir.Operation.make(ctx, "arith.convertf", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// arith.scaling_truncf — OCP MXFP downcast with per-block scale factors. +pub fn scaling_truncf( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + } + return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Compare +// ============================================================================= + +pub fn cmpi( + ctx: *mlir.Context, + predicate: CmpIPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpi", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + }, + .location = location, + }); +} + +pub fn cmpf( + ctx: *mlir.Context, + predicate: CmpFPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + }, + .location = location, + }); +} + +// ============================================================================= +// Select +// ============================================================================= + +/// arith.select — result type matches the true/false operand type. +pub fn select( + ctx: *mlir.Context, + cond: *const mlir.Value, + t: *const mlir.Value, + f: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.select", .{ + .operands = .{ .flat = &.{ cond, t, f } }, + .results = .{ .flat = &.{t.type_()} }, + .location = location, + }); +} diff --git a/mlir/dialects/arith/capi.cc b/mlir/dialects/arith/capi.cc new file mode 100644 index 00000000..3d7359c2 --- /dev/null +++ b/mlir/dialects/arith/capi.cc @@ -0,0 +1,47 @@ +//===- capi.cc - C API for the arith dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirArithFastMathAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeFastMathFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::FastMathFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirArithIntegerOverflowAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeIntegerOverflowFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::IntegerOverflowFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} diff --git a/mlir/dialects/arith/capi.h b/mlir/dialects/arith/capi.h new file mode 100644 index 00000000..ec8962af --- /dev/null +++ b/mlir/dialects/arith/capi.h @@ -0,0 +1,33 @@ +//===- capi.h - C API for the arith dialect -----------------------*- C -*-===// +// +// C entry points for the `arith` dialect's flag attributes, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_ARITH_CAPI_H +#define MLIR_DIALECTS_ARITH_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// #arith.fastmath<...>: `value` is the `FastMathFlags` bit set; `Get` returns +// a null attribute when it has bits outside the enum. +MLIR_CAPI_EXPORTED bool mlirAttributeIsAArithFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirArithIntegerOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_ARITH_CAPI_H diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig new file mode 100644 index 00000000..844898ca --- /dev/null +++ b/mlir/dialects/arith/test.zig @@ -0,0 +1,142 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "arith.addf", + "arith.addi", + "arith.addui_extended", + "arith.andi", + "arith.bitcast", + "arith.ceildivsi", + "arith.ceildivui", + "arith.cmpf", + "arith.cmpi", + "arith.constant", + "arith.convertf", + "arith.divf", + "arith.divsi", + "arith.divui", + "arith.extf", + "arith.extsi", + "arith.extui", + "arith.floordivsi", + "arith.flush_denormals", + "arith.fptosi", + "arith.fptoui", + "arith.index_cast", + "arith.index_castui", + "arith.maximumf", + "arith.maxnumf", + "arith.maxsi", + "arith.maxui", + "arith.minimumf", + "arith.minnumf", + "arith.minsi", + "arith.minui", + "arith.mulf", + "arith.muli", + "arith.mulsi_extended", + "arith.mului_extended", + "arith.negf", + "arith.ori", + "arith.remf", + "arith.remsi", + "arith.remui", + "arith.scaling_truncf", + "arith.select", + "arith.shli", + "arith.shrsi", + "arith.shrui", + "arith.sitofp", + "arith.subf", + "arith.subi", + "arith.truncf", + "arith.trunci", + "arith.uitofp", + "arith.xori", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(arith); + inline for (comptime std.meta.declarations(arith)) |decl| { + const T = @field(arith, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "arith fastmath attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.FastMathFlags, []const u8 }{ + .{ .none, "#arith.fastmath" }, + .{ .fast, "#arith.fastmath" }, + .{ .{ .reassoc = true }, "#arith.fastmath" }, + .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, + .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, + }; + for (cases) |case| { + const attr = try arith.FastMathAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.FastMathAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + +test "fastmath names the attribute of a float operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.fastmath(ctx, .fast)})}); + try std.testing.expectEqualStrings("{fastmath = #arith.fastmath}", w.buffered()); +} + +test "overflowFlags names the attribute of an integer operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.overflowFlags(ctx, .{ .nsw = true })})}); + try std.testing.expectEqualStrings("{overflowFlags = #arith.overflow}", w.buffered()); +} + +test "arith overflow attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.IntegerOverflowFlags, []const u8 }{ + .{ .none, "#arith.overflow" }, + .{ .{ .nsw = true }, "#arith.overflow" }, + .{ .{ .nsw = true, .nuw = true }, "#arith.overflow" }, + }; + for (cases) |case| { + const attr = try arith.IntegerOverflowAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.IntegerOverflowAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.IntegerOverflowAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + arith.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/cf/BUILD.bazel similarity index 53% rename from mlir/dialects/BUILD.bazel rename to mlir/dialects/cf/BUILD.bazel index 26cca662..3d308f63 100644 --- a/mlir/dialects/BUILD.bazel +++ b/mlir/dialects/cf/BUILD.bazel @@ -1,23 +1,23 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "dialects", - srcs = [ - "func.zig", - ], - import_name = "mlir/dialects", - main = "dialects.zig", + name = "cf", + import_name = "mlir/dialects/cf", + main = "cf.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//mlir", - "//mlir/dialects/stablehlo", "//stdx", + "@llvm-project//mlir:CAPICF", ], ) zig_test( name = "test", + main = "test.zig", deps = [ - ":dialects", + ":cf", + "//mlir", ], ) diff --git a/mlir/dialects/cf/cf.zig b/mlir/dialects/cf/cf.zig new file mode 100644 index 00000000..9a63297c --- /dev/null +++ b/mlir/dialects/cf/cf.zig @@ -0,0 +1,168 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cf"); +} + +// ============================================================================= +// cf.assert +// ============================================================================= + +/// cf.assert %cond, "message" — aborts execution with `msg` when `cond` is +/// false. `cond` is i1. +pub fn assert( + ctx: *mlir.Context, + cond: *const mlir.Value, + msg: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cf.assert", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "msg", .string(ctx, msg)), + }, + .location = location, + }); +} + +// ============================================================================= +// cf.br +// ============================================================================= + +/// cf.br ^dest(operands...) — unconditional branch. Terminator of its block. +/// The types of `dest_operands` must match the block arguments of `dest`. +pub fn br( + ctx: *mlir.Context, + dest: *mlir.Block, + dest_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("cf.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + return mlir.Operation.init(&state) catch @panic("Failed to create cf.br"); +} + +// ============================================================================= +// cf.cond_br +// ============================================================================= + +/// cf.cond_br %cond, ^trueDest(trueOps...), ^falseDest(falseOps...) — branch +/// on an i1. Terminator of its block. +/// +/// Operand layout (flat) is `[cond, *trueOperands, *falseOperands]` with an +/// `operandSegmentSizes = [1, |trueOperands|, |falseOperands|]` attribute per +/// the `AttrSizedOperandSegments` trait on the op. +/// +/// `branch_weights` (optional) lets profile-guided lowering bias the branch; +/// when provided, its length must be 2 (true weight, false weight). +pub fn cond_br( + ctx: *mlir.Context, + cond: *const mlir.Value, + true_dest: *mlir.Block, + true_operands: []const *const mlir.Value, + false_dest: *mlir.Block, + false_operands: []const *const mlir.Value, + opts: struct { + branch_weights: ?[]const i32 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 128) = .empty; + operands_buf.appendAssumeCapacity(cond); + operands_buf.appendSliceAssumeCapacity(true_operands); + operands_buf.appendSliceAssumeCapacity(false_operands); + + const seg_sizes = [3]i32{ 1, @intCast(true_operands.len), @intCast(false_operands.len) }; + + var state: mlir.OperationState = .init("cf.cond_br", location); + state.addOperands(operands_buf.constSlice()); + var successors: [2]*const mlir.Block = .{ true_dest, false_dest }; + state.addSuccessors(&successors); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.branch_weights) |w| { + std.debug.assert(w.len == 2); + attrs.appendAssumeCapacity(.named(ctx, "branch_weights", .denseArray(ctx, .i32, w))); + } + state.addAttributes(attrs.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.cond_br"); +} + +// ============================================================================= +// cf.switch +// ============================================================================= + +/// cf.switch %flag : iN, [ default: ^def(defOps...), caseN: ^bbN(opsN...) ] +/// +/// `flag` is any integer type. `case_values` are matched signed; the matching +/// case's block is jumped to (with the corresponding operand segment +/// forwarded). If no case matches, `default_dest` is taken with +/// `default_operands`. +/// +/// Operand layout: `[flag, *defaultOperands, *caseOperands[0], ..., +/// *caseOperands[N-1]]`. `case_operand_segments[i]` carries the length of +/// each case's operand segment. The `AttrSizedOperandSegments` trait tracks +/// the top-level split between `defaultOperands` and the lumped +/// `caseOperands`. +pub fn switch_( + ctx: *mlir.Context, + flag: *const mlir.Value, + default_dest: *mlir.Block, + default_operands: []const *const mlir.Value, + case_values: []const i32, + case_dests: []const *mlir.Block, + case_operands: []const []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(case_values.len == case_dests.len); + std.debug.assert(case_values.len == case_operands.len); + + var operands_buf: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands_buf.appendAssumeCapacity(flag); + operands_buf.appendSliceAssumeCapacity(default_operands); + + var case_seg_sizes: stdx.BoundedArray(i32, 64) = .empty; + var case_total: i32 = 0; + for (case_operands) |segment| { + operands_buf.appendSliceAssumeCapacity(segment); + case_seg_sizes.appendAssumeCapacity(@intCast(segment.len)); + case_total += @intCast(segment.len); + } + + // Top-level AttrSizedOperandSegments: [flag=1, defaultOperands, caseOperands lumped]. + const top_seg_sizes = [3]i32{ 1, @intCast(default_operands.len), case_total }; + + var state: mlir.OperationState = .init("cf.switch", location); + state.addOperands(operands_buf.constSlice()); + + var successors_buf: stdx.BoundedArray(*const mlir.Block, 64) = .empty; + successors_buf.appendAssumeCapacity(default_dest); + for (case_dests) |d| successors_buf.appendAssumeCapacity(d); + state.addSuccessors(@constCast(successors_buf.constSlice())); + + // case_values is stored as a DenseIntElementsAttr (tensor); + // the td uses `AnyIntElementsAttr`, and the parser prints decimal literals + // matching the flag's bit width. We use i32 for the common path (matches + // Python Triton's flag type for pid compares). + const shaped = mlir.RankedTensorType.get( + &.{@intCast(case_values.len)}, + .int(ctx, .i32), + null, + ).shaped(); + + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &top_seg_sizes)), + .named(ctx, "case_values", .denseElements(shaped, case_values)), + .named(ctx, "case_operand_segments", .denseArray(ctx, .i32, case_seg_sizes.constSlice())), + }); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); +} diff --git a/mlir/dialects/cf/test.zig b/mlir/dialects/cf/test.zig new file mode 100644 index 00000000..5a91f8f4 --- /dev/null +++ b/mlir/dialects/cf/test.zig @@ -0,0 +1,31 @@ +const std = @import("std"); + +const cf = @import("mlir/dialects/cf"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cf.assert", + "cf.br", + "cf.cond_br", + "cf.switch", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cf); + inline for (comptime std.meta.declarations(cf)) |decl| { + const T = @field(cf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cuda/BUILD.bazel b/mlir/dialects/cuda/BUILD.bazel new file mode 100644 index 00000000..bcfabc4a --- /dev/null +++ b/mlir/dialects/cuda/BUILD.bazel @@ -0,0 +1,193 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/cuda", + main = "cuda.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cuda", + "//mlir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CudaAttrs.td", + "CudaDialect.td", + "CudaEnums.td", + "CudaOps.td", + "CudaTypes.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda", + ], + "CudaDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda", + ], + "CudaDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CudaEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CudaEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda", + ], + "CudaTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda", + ], + "CudaTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaTypes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CudaOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CudaOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CudaDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "CudaDialect.cpp", + "CudaInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cuda/CudaAttrs.td b/mlir/dialects/cuda/CudaAttrs.td new file mode 100644 index 00000000..4d5e6f08 --- /dev/null +++ b/mlir/dialects/cuda/CudaAttrs.td @@ -0,0 +1,68 @@ +//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL -*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// Every cuda attribute the compiler's parser knows. `device_attributes` and +// `func_attributes` print as `#cuda`: the compiler +// parses any dictionary there and prints back only the entries it keeps, +// whose keys were not recovered; these keep the dictionary as written, and +// print a space before it, which the compiler parses too. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUDA_CUDAATTRS_TD +#define MLIR_DIALECTS_CUDA_CUDAATTRS_TD + +include "mlir/dialects/cuda/CudaDialect.td" +include "mlir/dialects/cuda/CudaEnums.td" + +defvar Cuda_Ns = "::mlir::cutlass_compiler::cuda"; + +def Cuda_AssumeKernelAttr : Cuda_Attr<"AssumeKernel", "assume_kernel_attr"> { + let summary = "Whether `cuda.launch` may assume its callee is a kernel"; + let description = "`#cuda.assume_kernel_attr`"; + let parameters = (ins "bool":$value); + let assemblyFormat = "`<` $value `>`"; +} + +def Cuda_ComputeTargetAttr : Cuda_Attr<"ComputeTarget", "compute_target"> { + let summary = "Representation, portability and architectures of a compiled executable"; + let description = "`#cuda.compute_target`"; + let parameters = (ins + AttrParameter:$representation, + AttrParameter:$portability, + ArrayRefParameter:$archs + ); + let assemblyFormat = "`<` $representation `,` $portability `,` `[` $archs `]` `>`"; +} + +def Cuda_DevMaxSharedMemoryOptinAttr + : Cuda_Attr<"DevMaxSharedMemoryOptin", "dev_max_shared_memory_optin"> { + let summary = "The device's opt-in maximum of shared memory per block"; + let description = "`#cuda.dev_max_shared_memory_optin`"; +} + +def Cuda_DeviceAttributesAttr : Cuda_Attr<"DeviceAttributes", "device_attributes"> { + let summary = "CUDA device attributes"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +def Cuda_ExecutableAttr : Cuda_Attr<"Executable", "executable"> { + let summary = "Format of a compiled executable"; + let description = "`#cuda.executable`"; + let parameters = (ins AttrParameter:$format); + let assemblyFormat = "`<` $format `>`"; +} + +def Cuda_FuncAttributesAttr : Cuda_Attr<"FuncAttributes", "func_attributes"> { + let summary = "CUDA function attributes of a kernel"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +#endif // MLIR_DIALECTS_CUDA_CUDAATTRS_TD diff --git a/mlir/dialects/cuda/CudaDialect.cpp b/mlir/dialects/cuda/CudaDialect.cpp new file mode 100644 index 00000000..0ce2910d --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.cpp @@ -0,0 +1,38 @@ +//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL ------------===// + +#include "mlir/dialects/cuda/CudaDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cuda; + +#include "mlir/dialects/cuda/CudaDialect.cpp.inc" + +#include "mlir/dialects/cuda/CudaEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cuda/CudaOps.cpp.inc" + +void CudaDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/cuda/CudaOps.cpp.inc" + >(); +} diff --git a/mlir/dialects/cuda/CudaDialect.h b/mlir/dialects/cuda/CudaDialect.h new file mode 100644 index 00000000..a96cdd17 --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.h @@ -0,0 +1,30 @@ +//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ----*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_H +#define MLIR_DIALECTS_CUDA_CUDADIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mlir/dialects/cuda/CudaDialect.h.inc" + +#include "mlir/dialects/cuda/CudaEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cuda/CudaAttrs.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cuda/CudaTypes.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cuda/CudaOps.h.inc" + +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_H diff --git a/mlir/dialects/cuda/CudaDialect.td b/mlir/dialects/cuda/CudaDialect.td new file mode 100644 index 00000000..939e2d85 --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.td @@ -0,0 +1,38 @@ +//===- CudaDialect.td - CUDA host dialect of the CuTe DSL --*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_TD +#define MLIR_DIALECTS_CUDA_CUDADIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def Cuda_Dialect : Dialect { + let name = "cuda"; + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let summary = "NVIDIA's CUDA host-program dialect used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation, type, attribute and enum registry of + the proprietary `cuda` dialect embedded in the nvidia-cutlass-dsl 4.8.0 + compiler: kernels, launch configurations and the CUDA runtime and driver + calls a host function makes. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +class Cuda_Op traits = []> + : Op; + +class Cuda_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +class Cuda_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_TD diff --git a/mlir/dialects/cuda/CudaEnums.td b/mlir/dialects/cuda/CudaEnums.td new file mode 100644 index 00000000..f7980167 --- /dev/null +++ b/mlir/dialects/cuda/CudaEnums.td @@ -0,0 +1,336 @@ +// The cuda dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_cuda_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef MLIR_DIALECTS_CUDA_CUDAENUMS_TD +#define MLIR_DIALECTS_CUDA_CUDAENUMS_TD + +include "mlir/dialects/cuda/CudaDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cuda_ArchPortabilityEnum : I32EnumAttr<"ArchPortability", "Enumerated type describing portability", [ + I32EnumAttrCase<"portable", 0, "portable">, + I32EnumAttrCase<"conditional", 1, "conditional">, + I32EnumAttrCase<"family", 2, "family"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUDeviceAttrEnum : I32EnumAttr<"CUDeviceAttr", "Enumerated type describing CUDA device attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"maxThreadsPerBlock", 1, "maxThreadsPerBlock">, + I32EnumAttrCase<"maxBlockDimX", 2, "maxBlockDimX">, + I32EnumAttrCase<"maxBlockDimY", 3, "maxBlockDimY">, + I32EnumAttrCase<"maxBlockDimZ", 4, "maxBlockDimZ">, + I32EnumAttrCase<"maxGridDimX", 5, "maxGridDimX">, + I32EnumAttrCase<"maxGridDimY", 6, "maxGridDimY">, + I32EnumAttrCase<"maxGridDimZ", 7, "maxGridDimZ">, + I32EnumAttrCase<"maxSharedMemoryPerBlock", 8, "maxSharedMemoryPerBlock">, + I32EnumAttrCase<"totalConstantMemory", 9, "totalConstantMemory">, + I32EnumAttrCase<"warpSize", 10, "warpSize">, + I32EnumAttrCase<"maxPitch", 11, "maxPitch">, + I32EnumAttrCase<"maxRegistersPerBlock", 12, "maxRegistersPerBlock">, + I32EnumAttrCase<"clockRate", 13, "clockRate">, + I32EnumAttrCase<"textureAlignment", 14, "textureAlignment">, + I32EnumAttrCase<"gpuOverlap", 15, "gpuOverlap">, + I32EnumAttrCase<"multiProcessorCount", 16, "multiProcessorCount">, + I32EnumAttrCase<"kernelExecTimeout", 17, "kernelExecTimeout">, + I32EnumAttrCase<"integrated", 18, "integrated">, + I32EnumAttrCase<"canMapHostMemory", 19, "canMapHostMemory">, + I32EnumAttrCase<"computeMode", 20, "computeMode">, + I32EnumAttrCase<"maxTexture1DWidth", 21, "maxTexture1DWidth">, + I32EnumAttrCase<"maxTexture2DWidth", 22, "maxTexture2DWidth">, + I32EnumAttrCase<"maxTexture2DHeight", 23, "maxTexture2DHeight">, + I32EnumAttrCase<"maxTexture3DWidth", 24, "maxTexture3DWidth">, + I32EnumAttrCase<"maxTexture3DHeight", 25, "maxTexture3DHeight">, + I32EnumAttrCase<"maxTexture3DDepth", 26, "maxTexture3DDepth">, + I32EnumAttrCase<"maxTexture2DLayeredWidth", 27, "maxTexture2DLayeredWidth">, + I32EnumAttrCase<"maxTexture2DLayeredHeight", 28, "maxTexture2DLayeredHeight">, + I32EnumAttrCase<"maxTexture2DLayeredLayers", 29, "maxTexture2DLayeredLayers">, + I32EnumAttrCase<"surfaceAlignment", 30, "surfaceAlignment">, + I32EnumAttrCase<"concurrentKernels", 31, "concurrentKernels">, + I32EnumAttrCase<"eccEnabled", 32, "eccEnabled">, + I32EnumAttrCase<"pciBusId", 33, "pciBusId">, + I32EnumAttrCase<"pciDeviceId", 34, "pciDeviceId">, + I32EnumAttrCase<"tccDriver", 35, "tccDriver">, + I32EnumAttrCase<"memoryClockRate", 36, "memoryClockRate">, + I32EnumAttrCase<"globalMemoryBusWidth", 37, "globalMemoryBusWidth">, + I32EnumAttrCase<"l2CacheSize", 38, "l2CacheSize">, + I32EnumAttrCase<"maxThreadsPerMultiProcessor", 39, "maxThreadsPerMultiProcessor">, + I32EnumAttrCase<"asyncEngineCount", 40, "asyncEngineCount">, + I32EnumAttrCase<"unifiedAddressing", 41, "unifiedAddressing">, + I32EnumAttrCase<"maxTexture1DLayeredWidth", 42, "maxTexture1DLayeredWidth">, + I32EnumAttrCase<"maxTexture1DLayeredLayers", 43, "maxTexture1DLayeredLayers">, + I32EnumAttrCase<"maxTexture2DGatherWidth", 45, "maxTexture2DGatherWidth">, + I32EnumAttrCase<"maxTexture2DGatherHeight", 46, "maxTexture2DGatherHeight">, + I32EnumAttrCase<"maxTexture3DWidthAlt", 47, "maxTexture3DWidthAlt">, + I32EnumAttrCase<"maxTexture3DHeightAlt", 48, "maxTexture3DHeightAlt">, + I32EnumAttrCase<"maxTexture3DDepthAlt", 49, "maxTexture3DDepthAlt">, + I32EnumAttrCase<"pciDomainId", 50, "pciDomainId">, + I32EnumAttrCase<"texturePitchAlignment", 51, "texturePitchAlignment">, + I32EnumAttrCase<"maxTextureCubemapWidth", 52, "maxTextureCubemapWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredWidth", 53, "maxTextureCubemapLayeredWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredLayers", 54, "maxTextureCubemapLayeredLayers">, + I32EnumAttrCase<"maxSurface1DWidth", 55, "maxSurface1DWidth">, + I32EnumAttrCase<"maxSurface2DWidth", 56, "maxSurface2DWidth">, + I32EnumAttrCase<"maxSurface2DHeight", 57, "maxSurface2DHeight">, + I32EnumAttrCase<"maxSurface3DWidth", 58, "maxSurface3DWidth">, + I32EnumAttrCase<"maxSurface3DHeight", 59, "maxSurface3DHeight">, + I32EnumAttrCase<"maxSurface3DDepth", 60, "maxSurface3DDepth">, + I32EnumAttrCase<"maxSurface1DLayeredWidth", 61, "maxSurface1DLayeredWidth">, + I32EnumAttrCase<"maxSurface1DLayeredLayers", 62, "maxSurface1DLayeredLayers">, + I32EnumAttrCase<"maxSurface2DLayeredWidth", 63, "maxSurface2DLayeredWidth">, + I32EnumAttrCase<"maxSurface2DLayeredHeight", 64, "maxSurface2DLayeredHeight">, + I32EnumAttrCase<"maxSurface2DLayeredLayers", 65, "maxSurface2DLayeredLayers">, + I32EnumAttrCase<"maxSurfaceCubemapWidth", 66, "maxSurfaceCubemapWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredWidth", 67, "maxSurfaceCubemapLayeredWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredLayers", 68, "maxSurfaceCubemapLayeredLayers">, + I32EnumAttrCase<"maxTexture1DLinearWidth", 69, "maxTexture1DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearWidth", 70, "maxTexture2DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearHeight", 71, "maxTexture2DLinearHeight">, + I32EnumAttrCase<"maxTexture2DLinearPitch", 72, "maxTexture2DLinearPitch">, + I32EnumAttrCase<"maxTexture2DMipmappedWidth", 73, "maxTexture2DMipmappedWidth">, + I32EnumAttrCase<"maxTexture2DMipmappedHeight", 74, "maxTexture2DMipmappedHeight">, + I32EnumAttrCase<"computeCapabilityMajor", 75, "computeCapabilityMajor">, + I32EnumAttrCase<"computeCapabilityMinor", 76, "computeCapabilityMinor">, + I32EnumAttrCase<"maxTexture1DMipmappedWidth", 77, "maxTexture1DMipmappedWidth">, + I32EnumAttrCase<"streamPrioritiesSupported", 78, "streamPrioritiesSupported">, + I32EnumAttrCase<"globalL1CacheSupported", 79, "globalL1CacheSupported">, + I32EnumAttrCase<"localL1CacheSupported", 80, "localL1CacheSupported">, + I32EnumAttrCase<"maxSharedMemoryPerMultiprocessor", 81, "maxSharedMemoryPerMultiprocessor">, + I32EnumAttrCase<"maxRegistersPerMultiprocessor", 82, "maxRegistersPerMultiprocessor">, + I32EnumAttrCase<"managedMemory", 83, "managedMemory">, + I32EnumAttrCase<"isMultiGpuBoard", 84, "isMultiGpuBoard">, + I32EnumAttrCase<"multiGpuBoardGroupID", 85, "multiGpuBoardGroupID">, + I32EnumAttrCase<"hostNativeAtomicSupported", 86, "hostNativeAtomicSupported">, + I32EnumAttrCase<"singleToDoublePrecisionPerfRatio", 87, "singleToDoublePrecisionPerfRatio">, + I32EnumAttrCase<"pageableMemoryAccess", 88, "pageableMemoryAccess">, + I32EnumAttrCase<"concurrentManagedAccess", 89, "concurrentManagedAccess">, + I32EnumAttrCase<"computePreemptionSupported", 90, "computePreemptionSupported">, + I32EnumAttrCase<"canUseHostPointerForRegisteredMem", 91, "canUseHostPointerForRegisteredMem">, + I32EnumAttrCase<"reserved92", 92, "reserved92">, + I32EnumAttrCase<"reserved93", 93, "reserved93">, + I32EnumAttrCase<"reserved94", 94, "reserved94">, + I32EnumAttrCase<"cooperativeLaunch", 95, "cooperativeLaunch">, + I32EnumAttrCase<"reserved96", 96, "reserved96">, + I32EnumAttrCase<"maxSharedMemoryPerBlockOptin", 97, "maxSharedMemoryPerBlockOptin">, + I32EnumAttrCase<"canFlushRemoteWrites", 98, "canFlushRemoteWrites">, + I32EnumAttrCase<"hostRegisterSupported", 99, "hostRegisterSupported">, + I32EnumAttrCase<"pageableMemoryAccessUsesHostPageTables", 100, "pageableMemoryAccessUsesHostPageTables">, + I32EnumAttrCase<"directManagedMemAccessFromHost", 101, "directManagedMemAccessFromHost">, + I32EnumAttrCase<"maxBlocksPerMultiprocessor", 106, "maxBlocksPerMultiprocessor">, + I32EnumAttrCase<"maxPersistingL2CacheSize", 108, "maxPersistingL2CacheSize">, + I32EnumAttrCase<"maxAccessPolicyWindowSize", 109, "maxAccessPolicyWindowSize">, + I32EnumAttrCase<"reservedSharedMemoryPerBlock", 111, "reservedSharedMemoryPerBlock">, + I32EnumAttrCase<"sparseCudaArraySupported", 112, "sparseCudaArraySupported">, + I32EnumAttrCase<"hostRegisterReadOnlySupported", 113, "hostRegisterReadOnlySupported">, + I32EnumAttrCase<"timelineSemaphoreInteropSupported", 114, "timelineSemaphoreInteropSupported">, + I32EnumAttrCase<"memoryPoolsSupported", 115, "memoryPoolsSupported">, + I32EnumAttrCase<"gpuDirectRdmaSupported", 116, "gpuDirectRdmaSupported">, + I32EnumAttrCase<"gpuDirectRdmaFlushWritesOptions", 117, "gpuDirectRdmaFlushWritesOptions">, + I32EnumAttrCase<"gpuDirectRdmaWritesOrdering", 118, "gpuDirectRdmaWritesOrdering">, + I32EnumAttrCase<"memoryPoolSupportedHandleTypes", 119, "memoryPoolSupportedHandleTypes">, + I32EnumAttrCase<"clusterLaunch", 120, "clusterLaunch">, + I32EnumAttrCase<"deferredMappingCudaArraySupported", 121, "deferredMappingCudaArraySupported">, + I32EnumAttrCase<"reserved122", 122, "reserved122">, + I32EnumAttrCase<"reserved123", 123, "reserved123">, + I32EnumAttrCase<"reserved124", 124, "reserved124">, + I32EnumAttrCase<"ipcEventSupport", 125, "ipcEventSupport">, + I32EnumAttrCase<"memSyncDomainCount", 126, "memSyncDomainCount">, + I32EnumAttrCase<"reserved127", 127, "reserved127">, + I32EnumAttrCase<"reserved128", 128, "reserved128">, + I32EnumAttrCase<"reserved129", 129, "reserved129">, + I32EnumAttrCase<"numaConfig", 130, "numaConfig">, + I32EnumAttrCase<"numaId", 131, "numaId">, + I32EnumAttrCase<"reserved132", 132, "reserved132">, + I32EnumAttrCase<"mpsEnabled", 133, "mpsEnabled">, + I32EnumAttrCase<"hostNumaId", 134, "hostNumaId">, + I32EnumAttrCase<"d3d12CigSupported", 135, "d3d12CigSupported">, + I32EnumAttrCase<"vulkanCigSupported", 138, "vulkanCigSupported">, + I32EnumAttrCase<"gpuPciDeviceId", 139, "gpuPciDeviceId">, + I32EnumAttrCase<"gpuPciSubsystemId", 140, "gpuPciSubsystemId">, + I32EnumAttrCase<"reserved141", 141, "reserved141">, + I32EnumAttrCase<"hostNumaMemoryPoolsSupported", 142, "hostNumaMemoryPoolsSupported">, + I32EnumAttrCase<"hostNumaMultinodeIpcSupported", 143, "hostNumaMultinodeIpcSupported">, + I32EnumAttrCase<"hostMemoryPoolsSupported", 144, "hostMemoryPoolsSupported">, + I32EnumAttrCase<"reserved145", 145, "reserved145">, + I32EnumAttrCase<"onlyPartialHostNativeAtomicSupported", 147, "onlyPartialHostNativeAtomicSupported">, + I32EnumAttrCase<"cudaDevAttrOversizedSharedMemoryPerBlock", 150, "cudaDevAttrOversizedSharedMemoryPerBlock"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUFunctionAttributeEnum : I32EnumAttr<"CUFunctionAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"max_threads_per_block", 0, "max_threads_per_block">, + I32EnumAttrCase<"shared_size_bytes", 1, "shared_size_bytes">, + I32EnumAttrCase<"const_size_bytes", 2, "const_size_bytes">, + I32EnumAttrCase<"local_size_bytes", 3, "local_size_bytes">, + I32EnumAttrCase<"num_regs", 4, "num_regs">, + I32EnumAttrCase<"ptx_version", 5, "ptx_version">, + I32EnumAttrCase<"binary_version", 6, "binary_version">, + I32EnumAttrCase<"cache_mode_ca", 7, "cache_mode_ca">, + I32EnumAttrCase<"max_dynamic_shared_size_bytes", 8, "max_dynamic_shared_size_bytes">, + I32EnumAttrCase<"preferred_shared_memory_carveout", 9, "preferred_shared_memory_carveout">, + I32EnumAttrCase<"cluster_size_must_be_set", 10, "cluster_size_must_be_set">, + I32EnumAttrCase<"required_cluster_width", 11, "required_cluster_width">, + I32EnumAttrCase<"required_cluster_height", 12, "required_cluster_height">, + I32EnumAttrCase<"required_cluster_depth", 13, "required_cluster_depth">, + I32EnumAttrCase<"non_portable_cluster_size_allowed", 14, "non_portable_cluster_size_allowed">, + I32EnumAttrCase<"cluster_scheduling_policy_preference", 15, "cluster_scheduling_policy_preference"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaABIEnum : I32EnumAttr<"CudaABI", "Enumerated of the host library ABI", [ + I32EnumAttrCase<"LP64", 0, "LP64">, + I32EnumAttrCase<"ILP64", 1, "ILP64">, + I32EnumAttrCase<"LLP64", 2, "LLP64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaAccessPropertyEnum : I32EnumAttr<"CudaAccessProperty", "Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members", [ + I32EnumAttrCase<"cudaAccessPropertyNormal", 0, "cudaAccessPropertyNormal">, + I32EnumAttrCase<"cudaAccessPropertyStreaming", 1, "cudaAccessPropertyStreaming">, + I32EnumAttrCase<"cudaAccessPropertyPersisting", 2, "cudaAccessPropertyPersisting"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaClusterSchedulingPolicyEnum : I32EnumAttr<"CudaClusterSchedulingPolicy", "Enumerated type describing CUDA cluster scheduling policy", [ + I32EnumAttrCase<"cudaClusterSchedulingPolicyDefault", 0, "cudaClusterSchedulingPolicyDefault">, + I32EnumAttrCase<"cudaClusterSchedulingPolicySpread", 1, "cudaClusterSchedulingPolicySpread">, + I32EnumAttrCase<"cudaClusterSchedulingPolicyLoadBalancing", 2, "cudaClusterSchedulingPolicyLoadBalancing"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaFuncAttributeEnum : I32EnumAttr<"CudaFuncAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"cudaFuncAttributeSharedMemoryMode", 16, "cudaFuncAttributeSharedMemoryMode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchAttributeIDEnum : I32EnumAttr<"CudaLaunchAttributeID", "Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API", [ + I32EnumAttrCase<"cudaLaunchAttributeIgnore", 0, "cudaLaunchAttributeIgnore">, + I32EnumAttrCase<"cudaLaunchAttributeAccessPolicyWindow", 1, "cudaLaunchAttributeAccessPolicyWindow">, + I32EnumAttrCase<"cudaLaunchAttributeCooperative", 2, "cudaLaunchAttributeCooperative">, + I32EnumAttrCase<"cudaLaunchAttributeSynchronizationPolicy", 3, "cudaLaunchAttributeSynchronizationPolicy">, + I32EnumAttrCase<"cudaLaunchAttributeClusterDimension", 4, "cudaLaunchAttributeClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeClusterSchedulingPolicyPreference", 5, "cudaLaunchAttributeClusterSchedulingPolicyPreference">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticStreamSerialization", 6, "cudaLaunchAttributeProgrammaticStreamSerialization">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticEvent", 7, "cudaLaunchAttributeProgrammaticEvent">, + I32EnumAttrCase<"cudaLaunchAttributePriority", 8, "cudaLaunchAttributePriority">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomainMap", 9, "cudaLaunchAttributeMemSyncDomainMap">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomain", 10, "cudaLaunchAttributeMemSyncDomain">, + I32EnumAttrCase<"cudaLaunchAttributePreferredClusterDimension", 11, "cudaLaunchAttributePreferredClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeLaunchCompletionEvent", 12, "cudaLaunchAttributeLaunchCompletionEvent">, + I32EnumAttrCase<"cudaLaunchAttributeDeviceUpdatableKernelNode", 13, "cudaLaunchAttributeDeviceUpdatableKernelNode">, + I32EnumAttrCase<"cudaLaunchAttributePreferredSharedMemoryCarveout", 14, "cudaLaunchAttributePreferredSharedMemoryCarveout">, + I32EnumAttrCase<"cudaLaunchAttributeNvlinkUtilCentricScheduling", 16, "cudaLaunchAttributeNvlinkUtilCentricScheduling"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchMemSyncDomainEnum : I32EnumAttr<"CudaLaunchMemSyncDomain", "Enumerated type describing CUDA memory sync domain", [ + I32EnumAttrCase<"cudaLaunchMemSyncDomainDefault", 0, "cudaLaunchMemSyncDomainDefault">, + I32EnumAttrCase<"cudaLaunchMemSyncDomainRemote", 1, "cudaLaunchMemSyncDomainRemote"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSharedMemoryModeEnum : I32EnumAttr<"CudaSharedMemoryMode", "Enumerated type describing CUDA shared memory modes", [ + I32EnumAttrCase<"cudaSharedMemoryModeDefault", 0, "cudaSharedMemoryModeDefault">, + I32EnumAttrCase<"cudaSharedMemoryModeRequirePortable", 1, "cudaSharedMemoryModeRequirePortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowNonPortable", 2, "cudaSharedMemoryModeAllowNonPortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowOversizedSharedMemory", 3, "cudaSharedMemoryModeAllowOversizedSharedMemory">, + I32EnumAttrCase<"cudaSharedMemoryModePreferOversizedSharedMemory", 4, "cudaSharedMemoryModePreferOversizedSharedMemory"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSynchronizationPolicyEnum : I32EnumAttr<"CudaSynchronizationPolicy", "Enumerated type describing CUDA synchronization policy", [ + I32EnumAttrCase<"cudaSyncPolicyAuto", 0, "cudaSyncPolicyAuto">, + I32EnumAttrCase<"cudaSyncPolicySpin", 1, "cudaSyncPolicySpin">, + I32EnumAttrCase<"cudaSyncPolicyYield", 2, "cudaSyncPolicyYield">, + I32EnumAttrCase<"cudaSyncPolicyBlockingSync", 3, "cudaSyncPolicyBlockingSync"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableFormatEnum : I32EnumAttr<"ExecutableFormat", "Enumerated type describing the format of an executable", [ + I32EnumAttrCase<"text", 0, "text">, + I32EnumAttrCase<"elf", 1, "elf">, + I32EnumAttrCase<"bitcode", 2, "bitcode">, + I32EnumAttrCase<"fatbin", 3, "fatbin"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableRepresentationEnum : I32EnumAttr<"ExecutableRepresentation", "Enumerated type describing the representation of an executable", [ + I32EnumAttrCase<"sass", 0, "sass">, + I32EnumAttrCase<"ptx", 2, "ptx">, + I32EnumAttrCase<"nvvm", 3, "nvvm">, + I32EnumAttrCase<"ltoir", 4, "ltoir">, + I32EnumAttrCase<"mlir", 5, "mlir">, + I32EnumAttrCase<"llvm", 6, "llvm">, + I32EnumAttrCase<"hostobj", 7, "hostobj"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_GpuArchitectureEnum : I32EnumAttr<"GpuArchitecture", "Enumerated type denoting a GPU Architecture", [ + I32EnumAttrCase<"sm_80", 80, "sm_80">, + I32EnumAttrCase<"sm_86", 86, "sm_86">, + I32EnumAttrCase<"sm_87", 87, "sm_87">, + I32EnumAttrCase<"sm_88", 88, "sm_88">, + I32EnumAttrCase<"sm_89", 89, "sm_89">, + I32EnumAttrCase<"sm_90", 90, "sm_90">, + I32EnumAttrCase<"sm_100", 100, "sm_100">, + I32EnumAttrCase<"sm_101", 101, "sm_101">, + I32EnumAttrCase<"sm_103", 103, "sm_103">, + I32EnumAttrCase<"sm_107", 107, "sm_107">, + I32EnumAttrCase<"sm_110", 110, "sm_110">, + I32EnumAttrCase<"sm_120", 120, "sm_120">, + I32EnumAttrCase<"sm_121", 121, "sm_121"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_MemcpyKindEnum : I32EnumAttr<"MemcpyKind", "Enumerated type describing memcpy directions", [ + I32EnumAttrCase<"HostToHost", 0, "HostToHost">, + I32EnumAttrCase<"HostToDevice", 1, "HostToDevice">, + I32EnumAttrCase<"DeviceToHost", 2, "DeviceToHost">, + I32EnumAttrCase<"DeviceToDevice", 3, "DeviceToDevice">, + I32EnumAttrCase<"Default", 4, "Default"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_StreamCaptureModeEnumEnum : I32EnumAttr<"StreamCaptureModeEnum", "Enumerated type describing CUDA stream capture modes", [ + I32EnumAttrCase<"Global", 0, "Global">, + I32EnumAttrCase<"ThreadLocal", 1, "ThreadLocal">, + I32EnumAttrCase<"Relaxed", 2, "Relaxed"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +#endif // MLIR_DIALECTS_CUDA_CUDAENUMS_TD diff --git a/mlir/dialects/cuda/CudaInference.cpp b/mlir/dialects/cuda/CudaInference.cpp new file mode 100644 index 00000000..1289679e --- /dev/null +++ b/mlir/dialects/cuda/CudaInference.cpp @@ -0,0 +1,17 @@ +//===- CudaInference.cpp - result types the compiler infers ---------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "mlir/dialects/cuda/CudaDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::cuda; + +LogicalResult LaunchCfgCreateFromStreamOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/cuda/CudaOps.td b/mlir/dialects/cuda/CudaOps.td new file mode 100644 index 00000000..991814c1 --- /dev/null +++ b/mlir/dialects/cuda/CudaOps.td @@ -0,0 +1,3072 @@ +// The cuda dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cuda_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// cuda.launch_cfg.create_from_stream +// +// Constraints not recovered: + +#ifndef MLIR_DIALECTS_CUDA_CUDAOPS_TD +#define MLIR_DIALECTS_CUDA_CUDAOPS_TD + +include "mlir/dialects/cuda/CudaAttrs.td" +include "mlir/dialects/cuda/CudaEnums.td" +include "mlir/dialects/cuda/CudaTypes.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cuda_AnyCUDAStructType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeDim3Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeExtentType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeKernelNodeParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeMemcpy3DParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePosType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeUuidType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueCooperativeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "any CUDA struct type">; +def Cuda_AnyType : Type, "any type">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDevicePropType::get($_builder.getContext())">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped2 : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped2 : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeExtentType::get($_builder.getContext())">; +def Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped : Type($_self)">, "CUDA cudaKernelNodeParams struct type (value-typed).">; +def Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped : Type($_self)">, "CUDA cudaMemcpy3DParms struct type (value-typed).">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType::get($_builder.getContext())">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">; +def Cuda_CUDACudaPosStructTypeValueTyped : Type($_self)">, "CUDA cudaPos struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePosType::get($_builder.getContext())">; +def Cuda_CUDACudaPosStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPos struct type (value-typed).">; +def Cuda_CUDACudaUUIDTStructTypeValue : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeUuidType::get($_builder.getContext())">; +def Cuda_CUDACudaUUIDTStructTypeValue2 : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped : Type($_self)">, "CUDA dim3 struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped2 : Type($_self)">, "CUDA dim3 struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDim3Type::get($_builder.getContext())">; +def Cuda_CUDAGraph : Type($_self)">, "CUDA Graph">; +def Cuda_CUDAGraph2 : Type($_self)">, "CUDA Graph">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution : Type($_self)">, "CUDA Graph Execution">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphExecType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution2 : Type($_self)">, "CUDA Graph Execution">; +def Cuda_CUDAGraphNodeHandle : Type($_self)">, "CUDA Graph Node handle">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphNodeType::get($_builder.getContext())">; +def Cuda_CUDAGraphNodeHandle2 : Type($_self)">, "CUDA Graph Node handle">; +def Cuda_CUDAKernelHandle : Type($_self)">, "CUDA Kernel handle">, BuildableType<"::mlir::cutlass_compiler::cuda::KernelType::get($_builder.getContext())">; +def Cuda_CUDAKernelHandle2 : Type($_self)">, "CUDA Kernel handle">; +def Cuda_CUDALibraryHandle : Type($_self)">, "CUDA Library handle">; +def Cuda_CUDALibraryHandle2 : Type($_self)">, "CUDA Library handle">, BuildableType<"::mlir::cutlass_compiler::cuda::LibraryType::get($_builder.getContext())">; +def Cuda_CUDAModuleHandle : Type($_self)">, "CUDA Module handle">, BuildableType<"::mlir::cutlass_compiler::cuda::ModuleType::get($_builder.getContext())">; +def Cuda_CUDAResultType : Type($_self)">, "CUDA Result type">, BuildableType<"::mlir::cutlass_compiler::cuda::ResultType::get($_builder.getContext())">; +def Cuda_CUDAResultType2 : Type($_self)">, "CUDA Result type">; +def Cuda_CUDAResultTypeOrCUDAStream : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::StreamType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::ResultType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::EventType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::GraphDeviceNodeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::TensorMapType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::IntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::LaunchConfigType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "CUDA Result type or CUDA Stream or `Event` represents CUDA event or CUDA Tensor Map type or Representation of CUDA's extensible launch configuration mirroring cuda runtime API or `GraphDeviceNode` represents CUDA device node handle for device-side node update or `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth or 32-bit signless integer or 64-bit signless integer or pointer type or LLVM pointer type">; +def Cuda_CUDAStream : Type($_self)">, "CUDA Stream">; +def Cuda_CUDAStream2 : Type($_self)">, "CUDA Stream">, BuildableType<"::mlir::cutlass_compiler::cuda::StreamType::get($_builder.getContext())">; +def Cuda_CUDATensorMapType : Type($_self)">, "CUDA Tensor Map type">, BuildableType<"::mlir::cutlass_compiler::cuda::TensorMapType::get($_builder.getContext())">; +def Cuda_CudaIntIsAIntegerType : Type($_self)">, "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth">; +def Cuda_EventRepresentsCUDAEvent : Type($_self)">, "`Event` represents CUDA event">, BuildableType<"::mlir::cutlass_compiler::cuda::EventType::get($_builder.getContext())">; +def Cuda_EventRepresentsCUDAEvent2 : Type($_self)">, "`Event` represents CUDA event">; +def Cuda_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 3">; +def Cuda_FixedLengthVectorOf32Bit2 : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 2">; +def Cuda_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 8">; +def Cuda_FixedLengthVectorOf8Bit2 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 256">; +def Cuda_FixedLengthVectorOf8Bit3 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 16">; +def Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle : Type($_self)">, "`GraphDeviceNode` represents CUDA device node handle for device-side node update">; +def Cuda_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def Cuda_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def Cuda_PointerType : Type($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "pointer type">; +def Cuda_RepresentationOfCUDASExtensibleLaunch : Type($_self)">, "Representation of CUDA's extensible launch configuration mirroring cuda runtime API">; +def Cuda_T1BitSignlessInteger : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def Cuda_T32BitFloat : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def Cuda_T32BitFloat2 : Type($_self)">, "32-bit float">; +def Cuda_T32BitSignlessInteger : Type, "32-bit signless integer">; +def Cuda_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def Cuda_T64BitSignlessInteger : Type, "64-bit signless integer">; +def Cuda_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def Cuda_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "Variant `cooperative` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `syncPolicy` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaSynchronizationPolicy). or Variant `clusterSchedulingPolicyPreference` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaClusterSchedulingPolicy). or Variant `programmaticStreamSerializationAllowed` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `priority` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `memSyncDomain` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchMemSyncDomain). or Variant `sharedMemCarveout` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `nvlinkUtilCentricScheduling` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `portableClusterSizeMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchAttributePortableClusterMode). or Variant `sharedMemoryMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaSharedMemoryMode).">; + +// Attribute constraints the bindings name but do not describe. +def Cuda_DenseResourceElementsAttr : Attr($_self)">, "dense resource elements attribute"> { + let storageType = "::mlir::DenseResourceElementsAttr"; + let returnType = "::mlir::DenseResourceElementsAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_FuncFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_KernelFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_IntegerAttr : Attr($_self)">, "integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (CudaInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class Cuda_InferredOp traits = []> + : Cuda_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cuda_AddressOfOp : Cuda_Op<"addressof"> { + let arguments = (ins + SymbolRefAttr:$symname + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_BinaryOp : Cuda_Op<"binary", [NoTerminator]> { + let description = [{ + This operation represents a CUDA binary, which contains a number of functions and + a serialized representation of the binary. + + Examples: + + ```mlir + + cuda.binary @binary_name <...> { + cuda.kernel @function_name function_type + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_DenseResourceElementsAttr:$binary + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_CastOp : Cuda_Op<"cast", [Pure]> { + let description = [{ + Casts between CUDA types and standard dialect or built-in types. + The source/destination types are statically checked and the IR + is invalid if the cast is not possible. + + Examples: + ```mlir + + %res_i32 = cuda.cast %res : cuda.result -> i32 + %res1 = cuda.cast %res_i32 : i32 -> cuda.result + + %s_ptr = cuda.cast %hstream : cuda.stream -> ptr.ptr + %s_i64 = cuda.cast %hstream : cuda.stream -> i64 + ``` + }]; + let arguments = (ins + Cuda_CUDAResultTypeOrCUDAStream:$src + ); + let results = (outs + Cuda_CUDAResultTypeOrCUDAStream:$dst + ); +} +def Cuda_ComposeOp : Cuda_Op<"compose"> { + let description = [{ + Generic constructor for any auto-generated CUDA struct type. The result + type identifies which struct is being built; operands are the struct's + non-padding fields in declaration order. The verifier checks operand + count and per-operand type against the result type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to `llvm.mlir.undef` + a chain of `llvm.insertvalue` (one per + operand-mapped slot from `getOperandFieldIndices()`). Pure value SSA; + no memory. + + Example: + ```mlir + %pos = cuda.compose %x, %y, %z + : i64, i64, i64 -> !cuda.runtime.pos + ``` + }]; + let arguments = (ins + Variadic:$fields + ); + let results = (outs + Cuda_AnyCUDAStructType:$result + ); +} +def Cuda_DecomposeOp : Cuda_Op<"decompose"> { + let description = [{ + Generic accessor: returns each non-padding field of a CUDA struct value + as an SSA value, in declaration order. The verifier checks result count + and per-result type against the source type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to a chain of `llvm.extractvalue` (one per operand-mapped slot + from `getOperandFieldIndices()`). Pure value SSA; no memory. + + Example: + ```mlir + %x, %y, %z = cuda.decompose %pos + : !cuda.runtime.pos -> i64, i64, i64 + ``` + }]; + let arguments = (ins + Cuda_AnyCUDAStructType:$source + ); + let results = (outs + Optional:$fields + ); +} +def Cuda_DeviceSyncOp : Cuda_Op<"device.sync"> { + let description = [{ + Operation to synchronize with the device. See cudaDeviceSynchronize() + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventCreateOp : Cuda_Op<"driver.event.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$Flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$phEvent + ); +} +def Cuda_DriverEventDestroyOp : Cuda_Op<"driver.event.destroy"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventElapsedTimeOp : Cuda_Op<"driver.event.elapsed_time"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hStart, + Cuda_EventRepresentsCUDAEvent2:$hEnd + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$pMilliseconds + ); +} +def Cuda_DriverEventQueryOp : Cuda_Op<"driver.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordOp : Cuda_Op<"driver.event.record"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordWithFlagsOp : Cuda_Op<"driver.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventSynchronizeOp : Cuda_Op<"driver.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverLibraryGetGlobalOp : Cuda_Op<"driver.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetKernelOp : Cuda_Op<"driver.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_DriverLibraryGetKernelCountOp : Cuda_Op<"driver.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_DriverLibraryGetManagedOp : Cuda_Op<"driver.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetModuleOp : Cuda_Op<"driver.library.get_module", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAModuleHandle:$pMod + ); +} +def Cuda_DriverLibraryGetUnifiedFunctionOp : Cuda_Op<"driver.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_DriverLibraryLoadDataOp : Cuda_Op<"driver.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryLoadFromFileOp : Cuda_Op<"driver.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryUnloadOp : Cuda_Op<"driver.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverTensorMapEncodeIm2colOp : Cuda_Op<"driver.tensor_map.encode_im2col"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_PointerType:$pixelBoxLowerCorner, + Cuda_PointerType:$pixelBoxUpperCorner, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapEncodeIm2colWideOp : Cuda_Op<"driver.tensor_map.encode_im2col_wide"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_T32BitSignlessInteger:$pixelBoxLowerCornerWidth, + Cuda_T32BitSignlessInteger:$pixelBoxUpperCornerWidth, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$mode, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapReplaceAddressOp : Cuda_Op<"driver.tensor_map.replace_address"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$globalAddress + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_EventCreateOp : Cuda_Op<"event.create"> { + let description = [{ + Operation to create a CUDA event. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_EventDestroyOp : Cuda_Op<"event.destroy"> { + let description = [{ + Operation to destroy a CUDA event. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_EventElapsedTimeOp : Cuda_Op<"event.elapsed_time"> { + let description = [{ + Operation to compute the elapsed time between events + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$start_event, + Cuda_EventRepresentsCUDAEvent2:$end_event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$elapsed_time + ); +} +def Cuda_EventRecordOp : Cuda_Op<"event.record"> { + let description = [{ + Operation to record an event in a CUDA stream. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_FuncOp : Cuda_Op<"func"> { + let description = [{ + This operation represents a CUDA device function which can be called from + another `cuda.func` or a `cuda.kernel`. It cannot be launched from the host. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_FuncFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_FuncGetNameOp : Cuda_Op<"func.get_name"> { + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel + ); + let results = (outs + Cuda_CUDAResultType2:$result, + Cuda_PointerType:$name + ); +} +def Cuda_GraphDestroyOp : Cuda_Op<"graph.destroy"> { + let description = [{ + Operation to destroy a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphInstantiateOp : Cuda_Op<"graph.instantiate"> { + let description = [{ + Operation to instantiate a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraphExecution:$graph_exec + ); +} +def Cuda_GraphLaunchOp : Cuda_Op<"graph.launch"> { + let description = [{ + Operation to launch a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphExecDestroyOp : Cuda_Op<"graph_exec.destroy"> { + let description = [{ + Operation to destroy a CUDA graph execution. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_KernelOp : Cuda_Op<"kernel"> { + let description = [{ + This operation represents a CUDA kernel which may be launched as a grid. It + differs from a `cuda.func`. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) + attributes { + cu_attrs = { + num_regs = 128, + max_dynamic_shared_size_bytes = 2048 + // ... add more cuda function attributes here + } + } + } + + ... + + %res = cuda.launch @module_name::@device_func <<< %grid, %block, %cluster, %smem, %stream >>>(%0) + + %0 = cuda.result.succeeded(%res) + scf.if %0 { + + } + else { + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_KernelFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$cu_func_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_LaunchOp : Cuda_Op<"launch", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a `cuda.kernel` on the device. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + + %0 = ... // kernel arguments + + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchCfgAccessPolicyWindowOp : Cuda_Op<"launch_cfg.access_policy_window"> { + let description = [{ + Sets accessPolicyWindow for a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.access_policy_window [%cfg0] ( + base_ptr=%basePtr, + hitProp=%hitProp, + hitRatio=%hitRatio, + missProp=%missProp, + numBytes=%numBytes + ) : !cuda.launch_cfg, !ptr.ptr<#ptr.generic_space>, f32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_PointerType:$base_ptr, + Cuda_T32BitFloat2:$hitRatio, + Cuda_T64BitSignlessInteger:$num_bytes, + Cuda_CudaAccessPropertyEnum:$hitProp, + Cuda_CudaAccessPropertyEnum:$missProp + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterDimOp : Cuda_Op<"launch_cfg.cluster_dim"> { + let description = [{ + Sets clusterDim for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.cluster_dim [%cfg0] (1, 2, 8) : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$clusterDimX, + Cuda_T32BitSignlessInteger:$clusterDimY, + Cuda_T32BitSignlessInteger:$clusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterSchedulingPolicyOp : Cuda_Op<"launch_cfg.cluster_scheduling_policy"> { + let description = [{ + Sets clusterSchedulingPolicyPreference for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cluster_scheduling_policy [%cfg0] cudaClusterSchedulingPolicyLoadBalancing + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaClusterSchedulingPolicyEnum:$policy + ); + let results = (outs); +} +def Cuda_LaunchCfgCooperativeOp : Cuda_Op<"launch_cfg.cooperative"> { + let description = [{ + Sets cooperative attribute to a launch_config. Nonzero indicates a cooperative kernel. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cooperative [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$cooperative + ); + let results = (outs); +} +def Cuda_LaunchCfgCreateOp : Cuda_Op<"launch_cfg.create"> { + let description = [{ + Creates a launch_config object with the specified runtime values. + The result is a Cuda_LaunchConfig type that contains all the runtime data. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create( + blockDim=(%block_x, %block_y, %block_z), + dynamicSmemBytes=%dynamic_smem_bytes, + gridDim=(%grid_x, %grid_y, %grid_z), + stream=%hstream + ) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgCreateFromStreamOp : Cuda_InferredOp<"launch_cfg.create_from_stream"> { + let description = [{ + Creates a launch_config object initialized with a CUDA stream and space for + the specified number of launch attributes. Grid dimensions, block + dimensions, and dynamic shared memory bytes can be supplied later with + `cuda.launch_cfg.grid_dimensions`. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create_from_stream( + stream=%hstream + ) : !cuda.stream + ``` + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgDeviceUpdatableKernelNodeOp : Cuda_Op<"launch_cfg.device_updatable_kernel_node"> { + let description = [{ + Sets deviceUpdatableKernelNode attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.device_updatable_kernel_node [%cfg0] (devNode=%devNode, deviceUpdatable=%deviceUpdatable) + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle:$devNode, + Cuda_T32BitSignlessInteger:$deviceUpdatable + ); + let results = (outs); +} +def Cuda_LaunchCfgGridDimensionsOp : Cuda_Op<"launch_cfg.grid_dimensions"> { + let description = [{ + Sets the fixed launch dimensions for a launch_config object. This updates + the direct `cudaLaunchConfig_t` fields, not the launch attribute array. + + Example: + ```mlir + cuda.launch_cfg.grid_dimensions [%cfg0] + grid(%gx, %gy, %gz), block(%bx, %by, %bz), dynamicSmem(%smem) + : !cuda.launch_cfg, i32, i32, i32, i32, i32, i32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes + ); + let results = (outs); +} +def Cuda_LaunchCfgLaunchCompletionEventOp : Cuda_Op<"launch_cfg.launch_completion_event"> { + let description = [{ + Sets launchCompletionEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.launch_completion_event [%cfg0] (event=%event, flags=%flags) + : !cuda.launch_cfg, !cuda.event, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainOp : Cuda_Op<"launch_cfg.mem_sync_domain"> { + let description = [{ + Sets memSyncDomain attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain [%cfg0] cudaMemSyncDomainRemote + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaLaunchMemSyncDomainEnum:$memSyncDomain + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainMapOp : Cuda_Op<"launch_cfg.mem_sync_domain_map"> { + let description = [{ + Sets memSyncDomainMap attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain_map [%cfg0] (domain_default=0, domain_remote=1) + : !cuda.launch_cfg, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$domain_default, + Cuda_T32BitSignlessInteger:$domain_remote + ); + let results = (outs); +} +def Cuda_LaunchCfgNvlinkUtilCentricSchedulingOp : Cuda_Op<"launch_cfg.nvlink_util_centric_scheduling"> { + let description = [{ + Sets nvlinkUtilCentricScheduling attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.nvlink_util_centric_scheduling [%cfg0] 1 : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$nvlinkSchedulingEnabled + ); + let results = (outs); +} +def Cuda_LaunchCfgPreferredClusterDimOp : Cuda_Op<"launch_cfg.preferred_cluster_dim"> { + let description = [{ + Sets preferredClusterDim attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.preferred_cluster_dim [%cfg0] (1, 1, 8) + : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$preferredClusterDimX, + Cuda_T32BitSignlessInteger:$preferredClusterDimY, + Cuda_T32BitSignlessInteger:$preferredClusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgPrintOp : Cuda_Op<"launch_cfg.print"> { + let description = [{ + Prints a cuda launch config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.print [%cfg0] : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg + ); + let results = (outs); +} +def Cuda_LaunchCfgPriorityOp : Cuda_Op<"launch_cfg.priority"> { + let description = [{ + Sets priority attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.priority [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$priority + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticEventOp : Cuda_Op<"launch_cfg.programmatic_event"> { + let description = [{ + Sets programmaticEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_event [%cfg0] ( + event=%event, + flags=%flags, + triggerAtBlockStart=%triggerAtBlockStart) + : !cuda.launch_cfg, !cuda.event, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags, + Cuda_T32BitSignlessInteger:$triggerAtBlockStart + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticStreamSerializationAllowedOp : Cuda_Op<"launch_cfg.programmatic_stream_serialization_allowed"> { + let description = [{ + Sets programmaticStreamSerializationAllowed attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_stream_serialization_allowed [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$programmaticStreamSerializationAllowed + ); + let results = (outs); +} +def Cuda_LaunchCfgSharedMemCarveoutOp : Cuda_Op<"launch_cfg.shared_mem_carveout"> { + let description = [{ + Sets sharedMemCarveout attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.shared_mem_carveout [%cfg0] 1024 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$sharedMemCarveout + ); + let results = (outs); +} +def Cuda_LaunchCfgSyncPolicyOp : Cuda_Op<"launch_cfg.sync_policy"> { + let description = [{ + Sets syncPolicy for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.sync_policy [%cfg0] cudaSynchronizationPolicyDefault + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaSynchronizationPolicyEnum:$syncPolicy + ); + let results = (outs); +} +def Cuda_LaunchExOp : Cuda_Op<"launch_ex"> { + let description = [{ + Operation to launch a `cuda.kernel` on the device using extensible launch API. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + %0 = ... // kernel arguments + + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchKernelOp : Cuda_Op<"launch_kernel", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a kernel from a kernel pointer on the device. This launch API + directly follows the CUDA C++ runtime API, passing all kernel arguments as a pointer + to a struct. + + Example: + + ```mlir + %res2 = cuda.launch_kernel %kernel, (%one32, %one32, %one32), (%one32, %one32, %one32), %zero64, %stream, %zeroPtr : !ptr.ptr<#ptr.generic_space> -> !cuda.result + + ``` + }]; + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel, + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Cuda_PointerType:$kernel_args, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LibraryGetKernelOp : Cuda_Op<"library.get_kernel"> { + let description = [{ + Return a kernel handle object if found in the given library. + + ## Example + + ``` + %result_0, %kernel = cuda.library.get_kernel %l1, %name : !cuda.library, !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.kernel + ``` + + ## Behavior + + This op executes the CUDA API function cudaGetKernel. If the kernel is not found the cuda result will be `cudaErrorSymbolNotFound`. If the `kernel_name` is not a valid, null-terminated ASCII string, the behavior is UNDEFINED. Upon a result of `cudaSuccess`, `kernel` contains a valid kernel handle object. Is the result is not `cudaSuccess`, and use of the `kernel_name` handle is UNDEFINED. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$kernel_name + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAKernelHandle:$kernel + ); +} +def Cuda_LibraryLoadDataOp : Cuda_Op<"library.load_data"> { + let description = [{ + Load a library from a pointer. + + ## Example + + ``` + %result,%l1 = cuda.library.load_data %modptr : !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.library + ``` + ## Behavior + + This op execute the CUDA API function cudaLibraryLoadData. Whether the library is loaded immediately depends on the module load environment. + }]; + let arguments = (ins + Cuda_PointerType:$data + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_LibraryUnloadOp : Cuda_Op<"library.unload"> { + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_MemoryAllocOp : Cuda_Op<"memory.alloc"> { + let description = [{ + Operation to allocate device memory. Calls `cuMemAlloc()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocAsyncOp : Cuda_Op<"memory.alloc_async"> { + let description = [{ + Operation to allocate device memory asynchronously. Calls `cuMemAllocAsync()`. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocHostOp : Cuda_Op<"memory.alloc_host"> { + let description = [{ + Operation to allocate host memory. Calls `cuMemAllocHost()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocManagedOp : Cuda_Op<"memory.alloc_managed"> { + let description = [{ + Operation to allocate managed memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %managed_mem, %res = cuda.memory.alloc_managed(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%managed_mem) + + %res1 = cuda.memory.free(%managed_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryCopyOp : Cuda_Op<"memory.copy"> { + let description = [{ + Operation to copy memory between host and device. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryCopyAsyncOp : Cuda_Op<"memory.copy_async"> { + let description = [{ + Operation to copy memory between host and device asynchronously. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy_async %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy_async %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeOp : Cuda_Op<"memory.free"> { + let description = [{ + Operation to free allocated device memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeAsyncOp : Cuda_Op<"memory.free_async"> { + let description = [{ + Operation to free allocated device memory asynchronously. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + + Preserves semantics of [cuMemAllocAsync](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MALLOC__ASYNC.html#group__CUDA__MALLOC__ASYNC_1g13413273e84a641bce1929eae9e6501f) + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeHostOp : Cuda_Op<"memory.free_host"> { + let description = [{ + Operation to free allocated host memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetOp : Cuda_Op<"memory.set"> { + let description = [{ + Operation to fill memory with a uniform value. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetAsyncOp : Cuda_Op<"memory.set_async"> { + let description = [{ + Operation to fill memory with a uniform value asynchronously. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_OccupancyMaxActiveBlocks : Cuda_Op<"occupancy_max_active_blocks_per_multiprocessor"> { + let description = [{ + Returns in `num_blocks` the maximum number of active blocks per streaming multiplrocessor for the device function. + + Example: + + ```mlir + %one64 = arith.constant 1 : i64 + %one32 = arith.constant 1 : i32 + %res, %num_blocks = cuda.occupancy_max_active_blocks_per_multiprocessor @funcname, %one32, %one64 : (i32, i64) -> (!cuda.result, i32) + ``` + }]; + let arguments = (ins + Cuda_CudaIntIsAIntegerType:$block_size, + Cuda_T64BitSignlessInteger:$dynamic_smem_size, + SymbolRefAttr:$func + ); + let results = (outs + Cuda_CUDAResultType:$res, + Cuda_T32BitSignlessInteger2:$num_blocks + ); +} +def Cuda_ProfilerStart : Cuda_Op<"profiler.start"> { + let description = [{ + Starts the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ProfilerStop : Cuda_Op<"profiler.stop"> { + let description = [{ + Stops the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ResultAssertSuccessOp : Cuda_Op<"result.assert_success"> { + let description = [{ + Asserts that a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultFailedOp : Cuda_Op<"result.failed"> { + let description = [{ + Determines if a `cuda.result` indicates failure. + }]; + let arguments = (ins); + let results = (outs); +} +def Cuda_ResultPrintOnErrorOp : Cuda_Op<"result.print_on_error"> { + let description = [{ + Prints a message on error. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultSucceededOp : Cuda_Op<"result.succeeded"> { + let description = [{ + Determines if a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs + Cuda_T1BitSignlessInteger:$success + ); +} +def Cuda_ReturnOp : Cuda_Op<"return", [Terminator]> { + let description = [{ + Returns a value from a `cuda.kernel` or `cuda.func` region. + + Note, the emitter seems to omit the dialect and simply prints `return` in the + custom assembly syntax. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_ReturnIfErrorOp : Cuda_Op<"return_if_error"> { + let description = [{ + The first operand must be the !cuda.result to check. + The subsequent operands are the values to be returned if an error exists. + This op is first lowered to an intermediate op `cuda.return_with_error`, + before final conversion to LLVM in a subsequent pass `cuda-finalize-error-handling`. + + Examples: + ```mlir + func.func @host_func() -> i32 { + %ten = arith.constant 10 : i32 + %cuda_result = cuda.launch @module::@kernel... + // Create an undefined/poison value for the error return path. + // Users can also input any arbitrary value for the error return path. + %poison = llvm.mlir.poison : i32 + // return poison if cuda_result is not cudaSuccess + %res_i32 = cuda.cast %cuda_result : !cuda.result -> i32 + cuda.return_if_error %res_i32, %poison : i32, i32 + + // return 10 if execution reaches here + return %ten : i32 + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_RuntimeGetDevicePropertiesOp : Cuda_Op<"runtime.device.properties"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$device + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDACudaDevicePropStructTypeValueTyped:$prop + ); +} +def Cuda_RuntimeDevicePropGetAccessPolicyMaxWindowSizeOp : Cuda_Op<"runtime.device_prop.get_access_policy_max_window_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetAsyncEngineCountOp : Cuda_Op<"runtime.device_prop.get_async_engine_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanMapHostMemoryOp : Cuda_Op<"runtime.device_prop.get_can_map_host_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanUseHostPointerForRegisteredMemOp : Cuda_Op<"runtime.device_prop.get_can_use_host_pointer_for_registered_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetClusterLaunchOp : Cuda_Op<"runtime.device_prop.get_cluster_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetComputePreemptionSupportedOp : Cuda_Op<"runtime.device_prop.get_compute_preemption_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentKernelsOp : Cuda_Op<"runtime.device_prop.get_concurrent_kernels", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentManagedAccessOp : Cuda_Op<"runtime.device_prop.get_concurrent_managed_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCooperativeLaunchOp : Cuda_Op<"runtime.device_prop.get_cooperative_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeferredMappingCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_deferred_mapping_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaConfigOp : Cuda_Op<"runtime.device_prop.get_device_numa_config", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaIdOp : Cuda_Op<"runtime.device_prop.get_device_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDirectManagedMemAccessFromHostOp : Cuda_Op<"runtime.device_prop.get_direct_managed_mem_access_from_host", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetECCEnabledOp : Cuda_Op<"runtime.device_prop.get_ecc_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGlobalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_global_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAFlushWritesOptionsOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMASupportedOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAWritesOrderingOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_writes_ordering", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciSubsystemIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_subsystem_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNativeAtomicSupportedOp : Cuda_Op<"runtime.device_prop.get_host_native_atomic_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaIdOp : Cuda_Op<"runtime.device_prop.get_host_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaMultinodeIpcSupportedOp : Cuda_Op<"runtime.device_prop.get_host_numa_multinode_ipc_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterReadOnlySupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_read_only_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterSupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIntegratedOp : Cuda_Op<"runtime.device_prop.get_integrated", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIpcEventSupportedOp : Cuda_Op<"runtime.device_prop.get_ipc_event_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIsMultiGpuBoardOp : Cuda_Op<"runtime.device_prop.get_is_multi_gpu_board", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetL2CacheSizeOp : Cuda_Op<"runtime.device_prop.get_l_2cache_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLocalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_local_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidOp : Cuda_Op<"runtime.device_prop.get_luid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidDeviceNodeMaskOp : Cuda_Op<"runtime.device_prop.get_luid_device_node_mask", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMajorOp : Cuda_Op<"runtime.device_prop.get_major", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetManagedMemoryOp : Cuda_Op<"runtime.device_prop.get_managed_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxBlocksPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_blocks_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxGridSizeOp : Cuda_Op<"runtime.device_prop.get_max_grid_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface3DOp : Cuda_Op<"runtime.device_prop.get_max_surface_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DGatherOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_gather", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLinearOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_linear", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DAltOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d_alt", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsDimOp : Cuda_Op<"runtime.device_prop.get_max_threads_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerBlockOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemPitchOp : Cuda_Op<"runtime.device_prop.get_mem_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryBusWidthOp : Cuda_Op<"runtime.device_prop.get_memory_bus_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolSupportedHandleTypesOp : Cuda_Op<"runtime.device_prop.get_memory_pool_supported_handle_types", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolsSupportedOp : Cuda_Op<"runtime.device_prop.get_memory_pools_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMinorOp : Cuda_Op<"runtime.device_prop.get_minor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMpsEnabledOp : Cuda_Op<"runtime.device_prop.get_mps_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiGpuBoardGroupIDOp : Cuda_Op<"runtime.device_prop.get_multi_gpu_board_group_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiProcessorCountOp : Cuda_Op<"runtime.device_prop.get_multi_processor_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetNameOp : Cuda_Op<"runtime.device_prop.get_name", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessUsesHostPageTablesOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciBusIDOp : Cuda_Op<"runtime.device_prop.get_pci_bus_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDomainIDOp : Cuda_Op<"runtime.device_prop.get_pci_domain_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPersistingL2CacheMaxSizeOp : Cuda_Op<"runtime.device_prop.get_persisting_l2cache_max_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerBlockOp : Cuda_Op<"runtime.device_prop.get_regs_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_regs_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetReservedSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_reserved_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOptinOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block_optin", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSparseCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_sparse_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetStreamPrioritiesSupportedOp : Cuda_Op<"runtime.device_prop.get_stream_priorities_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSurfaceAlignmentOp : Cuda_Op<"runtime.device_prop.get_surface_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTccDriverOp : Cuda_Op<"runtime.device_prop.get_tcc_driver", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTextureAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTexturePitchAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_pitch_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTimelineSemaphoreInteropSupportedOp : Cuda_Op<"runtime.device_prop.get_timeline_semaphore_interop_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalConstMemOp : Cuda_Op<"runtime.device_prop.get_total_const_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalGlobalMemOp : Cuda_Op<"runtime.device_prop.get_total_global_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedAddressingOp : Cuda_Op<"runtime.device_prop.get_unified_addressing", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedFunctionPointersOp : Cuda_Op<"runtime.device_prop.get_unified_function_pointers", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUuidOp : Cuda_Op<"runtime.device_prop.get_uuid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_CUDACudaUUIDTStructTypeValue:$result + ); +} +def Cuda_RuntimeDevicePropGetWarpSizeOp : Cuda_Op<"runtime.device_prop.get_warp_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetXOp : Cuda_Op<"runtime.dim_3.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetYOp : Cuda_Op<"runtime.dim_3.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetZOp : Cuda_Op<"runtime.dim_3.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeEventCreateWithFlagsOp : Cuda_Op<"runtime.event.create_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_RuntimeEventQueryOp : Cuda_Op<"runtime.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventRecordWithFlagsOp : Cuda_Op<"runtime.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_CUDAStream:$stream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventSynchronizeOp : Cuda_Op<"runtime.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeExtentGetDepthOp : Cuda_Op<"runtime.extent.get_depth", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetHeightOp : Cuda_Op<"runtime.extent.get_height", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetWidthOp : Cuda_Op<"runtime.extent.get_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeGraphAddKernelNodeOp : Cuda_Op<"runtime.graph.add_kernel_node"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph, + Cuda_PointerType:$pDependencies, + Cuda_T64BitSignlessInteger:$numDependencies, + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$pNodeParams + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraphNodeHandle:$pGraphNode + ); +} +def Cuda_RuntimeGraphCreateOp : Cuda_Op<"runtime.graph.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraph2:$pGraph + ); +} +def Cuda_RuntimeGraphKernelNodeGetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_get_attribute", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value_out + ); +} +def Cuda_RuntimeGraphKernelNodeSetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_set_attribute"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeKernelNodeParamsGetBlockDimOp : Cuda_Op<"runtime.kernel_node_params.get_block_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetExtraOp : Cuda_Op<"runtime.kernel_node_params.get_extra", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetFuncOp : Cuda_Op<"runtime.kernel_node_params.get_func", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetGridDimOp : Cuda_Op<"runtime.kernel_node_params.get_grid_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetKernelParamsOp : Cuda_Op<"runtime.kernel_node_params.get_kernel_params", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetSharedMemBytesOp : Cuda_Op<"runtime.kernel_node_params.get_shared_mem_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeLibraryGetGlobalOp : Cuda_Op<"runtime.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetKernelOp : Cuda_Op<"runtime.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_RuntimeLibraryGetKernelCountOp : Cuda_Op<"runtime.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_RuntimeLibraryGetManagedOp : Cuda_Op<"runtime.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetUnifiedFunctionOp : Cuda_Op<"runtime.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_RuntimeLibraryLoadDataOp : Cuda_Op<"runtime.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryLoadFromFileOp : Cuda_Op<"runtime.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryUnloadOp : Cuda_Op<"runtime.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetExtentOp : Cuda_Op<"runtime.memcpy_3d_params.get_extent", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaExtentStructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetKindOp : Cuda_Op<"runtime.memcpy_3d_params.get_kind", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DOp : Cuda_Op<"runtime.memory.cpy3d"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DAsyncOp : Cuda_Op<"runtime.memory.cpy3d_async"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimePitchedPtrGetPitchOp : Cuda_Op<"runtime.pitched_ptr.get_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetPtrOp : Cuda_Op<"runtime.pitched_ptr.get_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimePitchedPtrGetXsizeOp : Cuda_Op<"runtime.pitched_ptr.get_xsize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetYsizeOp : Cuda_Op<"runtime.pitched_ptr.get_ysize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetXOp : Cuda_Op<"runtime.pos.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetYOp : Cuda_Op<"runtime.pos.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetZOp : Cuda_Op<"runtime.pos.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeUuidGetBytesOp : Cuda_Op<"runtime.uuid.get_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaUUIDTStructTypeValue2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit3:$result + ); +} +def Cuda_StreamAddCallbackOp : Cuda_Op<"stream.add_callback"> { + let description = [{ + Enqueues a host-side callback on the given CUDA stream. The callback is provided + as a symbol to a `func.func` with ABI: + `func @cb(!ptr.ptr<#ptr.generic_space>, i32, !ptr.ptr<#ptr.generic_space>) -> ()` + i.e., (stream, status, userData) -> void. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_PointerType:$user_data, + FlatSymbolRefAttr:$callback + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamBeginCaptureOp : Cuda_Op<"stream.begin_capture"> { + let description = [{ + Operation to begin capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_StreamCaptureModeEnumEnum:$mode + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamCreateOp : Cuda_Op<"stream.create"> { + let description = [{ + Operation to create a CUDA stream. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAStream2:$stream + ); +} +def Cuda_StreamDestroyOp : Cuda_Op<"stream.destroy"> { + let description = [{ + Operation to destroy a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamEndCaptureOp : Cuda_Op<"stream.end_capture"> { + let description = [{ + Operation to end capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraph2:$graph + ); +} +def Cuda_StreamSynchronizeOp : Cuda_Op<"stream.synchronize", [MemoryEffects<[MemRead]>]> { + let description = [{ + Operation to synchronize with a CUDA Stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Optional:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_TensorMapEncodeTiledOp : Cuda_Op<"tensor_map_encode_tiled"> { + let description = [{ + Creates a descriptor for Tensor Memory Access (TMA) object specified by the parameters + describing a tiled region and returns it in tensorMap. + + This operation corresponds to the CUDA Driver API function: + `cuTensorMapEncodeTiled` + + Tensor map objects are only supported on devices of compute capability 9.0 or higher. + The tensorMap address must be aligned to 64 bytes. + + All parameters are provided as runtime SSA values. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_LLVMPointerInAddressSpace0:$globalDims, + Cuda_LLVMPointerInAddressSpace0:$globalStrides, + Cuda_LLVMPointerInAddressSpace0:$boxDims, + Cuda_LLVMPointerInAddressSpace0:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$cuTensorMap + ); +} + +#endif // MLIR_DIALECTS_CUDA_CUDAOPS_TD diff --git a/mlir/dialects/cuda/CudaTypes.td b/mlir/dialects/cuda/CudaTypes.td new file mode 100644 index 00000000..d149651e --- /dev/null +++ b/mlir/dialects/cuda/CudaTypes.td @@ -0,0 +1,212 @@ +//===- CudaTypes.td - CUDA host types of the CuTe DSL ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler (its typed +// Python type classes, its mnemonics, `_cutlass_ir.cu13` parser messages and +// the verifier's wording for each type); do not edit by hand. +// +// Every cuda type the compiler's parser knows. Only `launch_cfg` has a +// parameter; the C++ class names not given by the typed Python classes +// (`!cuda.int`, the graph handles, `arch_set`, `kernel`, `module` and the +// launch attribute values) follow the mnemonic. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUDA_CUDATYPES_TD +#define MLIR_DIALECTS_CUDA_CUDATYPES_TD + +include "mlir/dialects/cuda/CudaDialect.td" + +def Cuda_ArchSetType : Cuda_Type<"ArchSet", "arch_set"> { + let summary = "`!cuda.arch_set`"; + let description = "`!cuda.arch_set`"; + let assemblyFormat = ""; +} + +def Cuda_EventType : Cuda_Type<"Event", "event"> { + let summary = "`Event` represents CUDA event"; + let description = "`!cuda.event`"; + let assemblyFormat = ""; +} + +def Cuda_GraphType : Cuda_Type<"Graph", "graph"> { + let summary = "CUDA Graph"; + let description = "`!cuda.graph`"; + let assemblyFormat = ""; +} + +def Cuda_GraphDeviceNodeType : Cuda_Type<"GraphDeviceNode", "graph_device_node"> { + let summary = "`GraphDeviceNode` represents CUDA device node handle for device-side node update"; + let description = "`!cuda.graph_device_node`"; + let assemblyFormat = ""; +} + +def Cuda_GraphExecType : Cuda_Type<"GraphExec", "graph_exec"> { + let summary = "CUDA Graph Execution"; + let description = "`!cuda.graph_exec`"; + let assemblyFormat = ""; +} + +def Cuda_GraphNodeType : Cuda_Type<"GraphNode", "graph_node"> { + let summary = "CUDA Graph Node handle"; + let description = "`!cuda.graph_node`"; + let assemblyFormat = ""; +} + +def Cuda_IntType : Cuda_Type<"Int", "int"> { + let summary = "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth"; + let description = "`!cuda.int`"; + let assemblyFormat = ""; +} + +def Cuda_KernelType : Cuda_Type<"Kernel", "kernel"> { + let summary = "CUDA Kernel handle"; + let description = "`!cuda.kernel`"; + let assemblyFormat = ""; +} + +def Cuda_LaunchConfigType : Cuda_Type<"LaunchConfig", "launch_cfg"> { + let summary = "Representation of CUDA's extensible launch configuration mirroring cuda runtime API"; + let description = "`!cuda.launch_cfg`"; + let parameters = (ins "uint32_t":$maxNumAttrs); + let assemblyFormat = "`<` `max_attrs` `=` $maxNumAttrs `>`"; +} + +def Cuda_LibraryType : Cuda_Type<"Library", "library"> { + let summary = "CUDA Library handle"; + let description = "`!cuda.library`"; + let assemblyFormat = ""; +} + +def Cuda_ModuleType : Cuda_Type<"Module", "module"> { + let summary = "CUDA Module handle"; + let description = "`!cuda.module`"; + let assemblyFormat = ""; +} + +def Cuda_ResultType : Cuda_Type<"Result", "result"> { + let summary = "CUDA Result type"; + let description = "`!cuda.result`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDevicePropType : Cuda_Type<"RuntimeDeviceProp", "runtime.device_prop"> { + let summary = "CUDA cudaDeviceProp struct type (value-typed)."; + let description = "`!cuda.runtime.device_prop`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDim3Type : Cuda_Type<"RuntimeDim3", "runtime.dim_3"> { + let summary = "CUDA dim3 struct type (value-typed)."; + let description = "`!cuda.runtime.dim_3`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeExtentType : Cuda_Type<"RuntimeExtent", "runtime.extent"> { + let summary = "CUDA cudaExtent struct type (value-typed)."; + let description = "`!cuda.runtime.extent`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeKernelNodeParamsType : Cuda_Type<"RuntimeKernelNodeParams", "runtime.kernel_node_params"> { + let summary = "CUDA cudaKernelNodeParams struct type (value-typed)."; + let description = "`!cuda.runtime.kernel_node_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType : Cuda_Type<"RuntimeLaunchAttributeValueClusterSchedulingPolicyPreference", "runtime.launch_attribute_value.cluster_scheduling_policy_preference"> { + let summary = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let description = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueCooperativeType : Cuda_Type<"RuntimeLaunchAttributeValueCooperative", "runtime.launch_attribute_value.cooperative"> { + let summary = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let description = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueMemSyncDomainType : Cuda_Type<"RuntimeLaunchAttributeValueMemSyncDomain", "runtime.launch_attribute_value.mem_sync_domain"> { + let summary = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let description = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType : Cuda_Type<"RuntimeLaunchAttributeValueNvlinkUtilCentricScheduling", "runtime.launch_attribute_value.nvlink_util_centric_scheduling"> { + let summary = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let description = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePortableClusterSizeModeType : Cuda_Type<"RuntimeLaunchAttributeValuePortableClusterSizeMode", "runtime.launch_attribute_value.portable_cluster_size_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePriorityType : Cuda_Type<"RuntimeLaunchAttributeValuePriority", "runtime.launch_attribute_value.priority"> { + let summary = "`!cuda.runtime.launch_attribute_value.priority`"; + let description = "`!cuda.runtime.launch_attribute_value.priority`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType : Cuda_Type<"RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed", "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"> { + let summary = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let description = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemCarveoutType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemCarveout", "runtime.launch_attribute_value.shared_mem_carveout"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemoryModeType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemoryMode", "runtime.launch_attribute_value.shared_memory_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSyncPolicyType : Cuda_Type<"RuntimeLaunchAttributeValueSyncPolicy", "runtime.launch_attribute_value.sync_policy"> { + let summary = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let description = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeMemcpy3DParamsType : Cuda_Type<"RuntimeMemcpy3DParams", "runtime.memcpy_3d_params"> { + let summary = "CUDA cudaMemcpy3DParms struct type (value-typed)."; + let description = "`!cuda.runtime.memcpy_3d_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePitchedPtrType : Cuda_Type<"RuntimePitchedPtr", "runtime.pitched_ptr"> { + let summary = "CUDA cudaPitchedPtr struct type (value-typed)."; + let description = "`!cuda.runtime.pitched_ptr`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePosType : Cuda_Type<"RuntimePos", "runtime.pos"> { + let summary = "CUDA cudaPos struct type (value-typed)."; + let description = "`!cuda.runtime.pos`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeUuidType : Cuda_Type<"RuntimeUuid", "runtime.uuid"> { + let summary = "CUDA cudaUUID_t struct type (value-typed)."; + let description = "`!cuda.runtime.uuid`"; + let assemblyFormat = ""; +} + +def Cuda_StreamType : Cuda_Type<"Stream", "stream"> { + let summary = "CUDA Stream"; + let description = "`!cuda.stream`"; + let assemblyFormat = ""; +} + +def Cuda_TensorMapType : Cuda_Type<"TensorMap", "tensor_map"> { + let summary = "CUDA Tensor Map type"; + let description = "`!cuda.tensor_map`"; + let assemblyFormat = ""; +} + +#endif // MLIR_DIALECTS_CUDA_CUDATYPES_TD diff --git a/mlir/dialects/cuda/capi.cc b/mlir/dialects/cuda/capi.cc new file mode 100644 index 00000000..4cdff5ee --- /dev/null +++ b/mlir/dialects/cuda/capi.cc @@ -0,0 +1,559 @@ +//===- capi.cc - C API for NVIDIA's cuda dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cuda/CudaDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cuda, cuda, + mlir::cutlass_compiler::cuda::CudaDialect) + +// Attributes + +namespace cuda = mlir::cutlass_compiler::cuda; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::AssumeKernelAttr::get(ctx, value)); +} +bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getValue(); +} + +bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, + uint32_t representation, + uint32_t portability, + intptr_t numArchs, + const uint32_t *archs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto representationValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation( + representation); + if (!representationValue) + return {nullptr}; + auto portabilityValue = + ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); + if (!portabilityValue) + return {nullptr}; + llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; + for (intptr_t i = 0; i < numArchs; ++i) { + auto value = + ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); + if (!value) + return {nullptr}; + archsValue.push_back(*value); + } + return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, + *portabilityValue, archsValue)); +} +uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getRepresentation()); +} +uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getPortability()); +} +intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getArchs().size(); +} +uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { + return static_cast( + llvm::cast(unwrap(attr)).getArchs()[pos]); +} + +bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); +} + +bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto formatValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); + if (!formatValue) + return {nullptr}; + return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); +} +uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getFormat()); +} + +bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +// Types + +bool mlirTypeIsACudaArchSet(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaArchSetTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ArchSetType::get(ctx)); +} + +bool mlirTypeIsACudaEvent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaEventTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::EventType::get(ctx)); +} + +bool mlirTypeIsACudaGraphDeviceNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphDeviceNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraphExec(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphExecTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphExecType::get(ctx)); +} + +bool mlirTypeIsACudaGraphNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraph(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphType::get(ctx)); +} + +bool mlirTypeIsACudaInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaIntTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::IntType::get(ctx)); +} + +bool mlirTypeIsACudaKernel(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaKernelTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::KernelType::get(ctx)); +} + +bool mlirTypeIsACudaLaunchConfig(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LaunchConfigType::get(ctx, maxNumAttrs)); +} +uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type) { + return llvm::cast(unwrap(type)).getMaxNumAttrs(); +} + +bool mlirTypeIsACudaLibrary(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLibraryTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LibraryType::get(ctx)); +} + +bool mlirTypeIsACudaModule(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaModuleTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ModuleType::get(ctx)); +} + +bool mlirTypeIsACudaResult(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaResultTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ResultType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDevicePropType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDim3(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDim3Type::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeExtent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeExtentType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeKernelNodeParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType:: + get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueCooperativeType>(unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueCooperativeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueMemSyncDomainType>(unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueMemSyncDomainType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>( + unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get( + ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>( + unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValuePriorityType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type) { + return llvm::isa_and_nonnull< + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType:: + get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemoryModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSyncPolicyType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeMemcpy3DParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePitchedPtrType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePos(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePosTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePosType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeUuid(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeUuidType::get(ctx)); +} + +bool mlirTypeIsACudaStream(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaStreamTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::StreamType::get(ctx)); +} + +bool mlirTypeIsACudaTensorMap(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaTensorMapTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::TensorMapType::get(ctx)); +} diff --git a/mlir/dialects/cuda/capi.h b/mlir/dialects/cuda/capi.h new file mode 100644 index 00000000..126a87d8 --- /dev/null +++ b/mlir/dialects/cuda/capi.h @@ -0,0 +1,253 @@ +//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CAPI_H +#define MLIR_DIALECTS_CUDA_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a +// kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); +MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); + +// `#cuda.compute_target`: Representation, portability and architectures of a +// compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet( + MlirContext context, uint32_t representation, uint32_t portability, + intptr_t numArchs, const uint32_t *archs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); + +// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared +// memory per block +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); + +// `#cuda.device_attributes`: CUDA device attributes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); + +// `#cuda.executable`: Format of a compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, + uint32_t format); +MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); + +// `#cuda.func_attributes`: CUDA function attributes of a kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); + +// Types + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `!cuda.arch_set`: `!cuda.arch_set` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); + +// `!cuda.event`: `Event` represents CUDA event +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); + +// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node +// handle for device-side node update +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); + +// `!cuda.graph_exec`: CUDA Graph Execution +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); + +// `!cuda.graph_node`: CUDA Graph Node handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); + +// `!cuda.graph`: CUDA Graph +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); + +// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the +// underlying ABI integer bitwidth +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); + +// `!cuda.kernel`: CUDA Kernel handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); + +// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration +// mirroring cuda runtime API +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); + +// `!cuda.library`: CUDA Library handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); + +// `!cuda.module`: CUDA Module handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); + +// `!cuda.result`: CUDA Result type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); + +// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeDevicePropTypeGet(MlirContext context); + +// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); + +// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); + +// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cooperative` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.mem_sync_domain` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.priority` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_memory_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.sync_policy` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); + +// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); + +// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); + +// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); + +// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); + +// `!cuda.stream`: CUDA Stream +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); + +// `!cuda.tensor_map`: CUDA Tensor Map type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_CAPI_H diff --git a/mlir/dialects/cuda/cuda.zig b/mlir/dialects/cuda/cuda.zig new file mode 100644 index 00000000..2c74c920 --- /dev/null +++ b/mlir/dialects/cuda/cuda.zig @@ -0,0 +1,104 @@ +//! Builders for the CUDA host-program operations CuTe kernels emit, over the +//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cuda`, `ops`), +//! which the CuTe-DSL compiler registers. Each builder fixes the attributes the +//! kernels do not choose. + +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} + +pub fn streamType(ctx: *mlir.Context) *const mlir.Type { + return (ops.StreamType.get(ctx, .{}) catch unreachable).type_(); +} + +pub fn resultType(ctx: *mlir.Context) *const mlir.Type { + return (ops.ResultType.get(ctx, .{}) catch unreachable).type_(); +} + +pub fn launchConfigType(ctx: *mlir.Context, max_attrs: u32) *const mlir.Type { + return (ops.LaunchConfigType.get(ctx, .{ .maxNumAttrs = max_attrs }) catch unreachable).type_(); +} + +/// `#cuda.assume_kernel_attr`, whether `cuda.launch_ex` may assume its +/// callee is a kernel. +pub fn assumeKernelAttr(ctx: *mlir.Context, value: bool) *const mlir.Attribute { + return (ops.AssumeKernelAttr.get(ctx, .{ .value = value }) catch unreachable).attribute(); +} + +pub const KernelArgs = struct { + name: []const u8, + function_type: *const mlir.Type, + body: *mlir.Block, + arg_attrs: ?*const mlir.Attribute = null, + extra_attributes: []const mlir.NamedAttribute = &.{}, + location: *const mlir.Location, +}; + +/// `cuda.kernel`; `body` is filled afterwards. +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), .{args.body}, .{ .arg_attrs = args.arg_attrs }, args.location); + for (args.extra_attributes) |attr| op.setAttribute(attr); + return op; +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.return_(ctx, values, location); +} + +pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.cast(ctx, src, dst, location); +} + +pub const LaunchConfigCreateArgs = struct { + max_attrs: u32, + block: [3]*const mlir.Value, + dynamic_smem: *const mlir.Value, + grid: [3]*const mlir.Value, + stream: *const mlir.Value, + location: *const mlir.Location, +}; + +pub fn launch_cfg_create(ctx: *mlir.Context, args: LaunchConfigCreateArgs) *mlir.Operation { + return ops.launch_cfg_create( + ctx, + args.block[0], + args.block[1], + args.block[2], + args.dynamic_smem, + args.grid[0], + args.grid[1], + args.grid[2], + args.stream, + launchConfigType(ctx, args.max_attrs), + .int(ctx, .i32, args.max_attrs), + args.location, + ); +} + +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, config: *const mlir.Value, dims: [3]*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_cluster_dim(ctx, config, dims[0], dims[1], dims[2], location); +} + +pub fn launch_cfg_cooperative(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_cooperative(ctx, config, enabled, location); +} + +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_programmatic_stream_serialization_allowed(ctx, config, enabled, location); +} + +pub const LaunchExArgs = struct { + config: *const mlir.Value, + inputs: []const *const mlir.Value, + callee: *const mlir.Attribute, + assume_kernel_attr: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { + return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, .{ .assume_kernel_attr = args.assume_kernel_attr }, args.location); +} diff --git a/mlir/dialects/cuda/ops.zig b/mlir/dialects/cuda/ops.zig new file mode 100644 index 00000000..baeb0ded --- /dev/null +++ b/mlir/dialects/cuda/ops.zig @@ -0,0 +1,3346 @@ +//! Zig bindings of NVIDIA's CUDA host dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// The C dialect handle, `mlirGetDialectHandle__cuda__`. +pub const dialect_handle = "cuda"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Enumerated type describing portability. +pub const ArchPortability = enum(u32) { + portable = 0, + conditional = 1, + family = 2, +}; + +/// Enumerated type describing CUDA device attributes can be set through Cuda Dialect. +pub const CUDeviceAttr = enum(u32) { + maxThreadsPerBlock = 1, + maxBlockDimX = 2, + maxBlockDimY = 3, + maxBlockDimZ = 4, + maxGridDimX = 5, + maxGridDimY = 6, + maxGridDimZ = 7, + maxSharedMemoryPerBlock = 8, + totalConstantMemory = 9, + warpSize = 10, + maxPitch = 11, + maxRegistersPerBlock = 12, + clockRate = 13, + textureAlignment = 14, + gpuOverlap = 15, + multiProcessorCount = 16, + kernelExecTimeout = 17, + integrated = 18, + canMapHostMemory = 19, + computeMode = 20, + maxTexture1DWidth = 21, + maxTexture2DWidth = 22, + maxTexture2DHeight = 23, + maxTexture3DWidth = 24, + maxTexture3DHeight = 25, + maxTexture3DDepth = 26, + maxTexture2DLayeredWidth = 27, + maxTexture2DLayeredHeight = 28, + maxTexture2DLayeredLayers = 29, + surfaceAlignment = 30, + concurrentKernels = 31, + eccEnabled = 32, + pciBusId = 33, + pciDeviceId = 34, + tccDriver = 35, + memoryClockRate = 36, + globalMemoryBusWidth = 37, + l2CacheSize = 38, + maxThreadsPerMultiProcessor = 39, + asyncEngineCount = 40, + unifiedAddressing = 41, + maxTexture1DLayeredWidth = 42, + maxTexture1DLayeredLayers = 43, + maxTexture2DGatherWidth = 45, + maxTexture2DGatherHeight = 46, + maxTexture3DWidthAlt = 47, + maxTexture3DHeightAlt = 48, + maxTexture3DDepthAlt = 49, + pciDomainId = 50, + texturePitchAlignment = 51, + maxTextureCubemapWidth = 52, + maxTextureCubemapLayeredWidth = 53, + maxTextureCubemapLayeredLayers = 54, + maxSurface1DWidth = 55, + maxSurface2DWidth = 56, + maxSurface2DHeight = 57, + maxSurface3DWidth = 58, + maxSurface3DHeight = 59, + maxSurface3DDepth = 60, + maxSurface1DLayeredWidth = 61, + maxSurface1DLayeredLayers = 62, + maxSurface2DLayeredWidth = 63, + maxSurface2DLayeredHeight = 64, + maxSurface2DLayeredLayers = 65, + maxSurfaceCubemapWidth = 66, + maxSurfaceCubemapLayeredWidth = 67, + maxSurfaceCubemapLayeredLayers = 68, + maxTexture1DLinearWidth = 69, + maxTexture2DLinearWidth = 70, + maxTexture2DLinearHeight = 71, + maxTexture2DLinearPitch = 72, + maxTexture2DMipmappedWidth = 73, + maxTexture2DMipmappedHeight = 74, + computeCapabilityMajor = 75, + computeCapabilityMinor = 76, + maxTexture1DMipmappedWidth = 77, + streamPrioritiesSupported = 78, + globalL1CacheSupported = 79, + localL1CacheSupported = 80, + maxSharedMemoryPerMultiprocessor = 81, + maxRegistersPerMultiprocessor = 82, + managedMemory = 83, + isMultiGpuBoard = 84, + multiGpuBoardGroupID = 85, + hostNativeAtomicSupported = 86, + singleToDoublePrecisionPerfRatio = 87, + pageableMemoryAccess = 88, + concurrentManagedAccess = 89, + computePreemptionSupported = 90, + canUseHostPointerForRegisteredMem = 91, + reserved92 = 92, + reserved93 = 93, + reserved94 = 94, + cooperativeLaunch = 95, + reserved96 = 96, + maxSharedMemoryPerBlockOptin = 97, + canFlushRemoteWrites = 98, + hostRegisterSupported = 99, + pageableMemoryAccessUsesHostPageTables = 100, + directManagedMemAccessFromHost = 101, + maxBlocksPerMultiprocessor = 106, + maxPersistingL2CacheSize = 108, + maxAccessPolicyWindowSize = 109, + reservedSharedMemoryPerBlock = 111, + sparseCudaArraySupported = 112, + hostRegisterReadOnlySupported = 113, + timelineSemaphoreInteropSupported = 114, + memoryPoolsSupported = 115, + gpuDirectRdmaSupported = 116, + gpuDirectRdmaFlushWritesOptions = 117, + gpuDirectRdmaWritesOrdering = 118, + memoryPoolSupportedHandleTypes = 119, + clusterLaunch = 120, + deferredMappingCudaArraySupported = 121, + reserved122 = 122, + reserved123 = 123, + reserved124 = 124, + ipcEventSupport = 125, + memSyncDomainCount = 126, + reserved127 = 127, + reserved128 = 128, + reserved129 = 129, + numaConfig = 130, + numaId = 131, + reserved132 = 132, + mpsEnabled = 133, + hostNumaId = 134, + d3d12CigSupported = 135, + vulkanCigSupported = 138, + gpuPciDeviceId = 139, + gpuPciSubsystemId = 140, + reserved141 = 141, + hostNumaMemoryPoolsSupported = 142, + hostNumaMultinodeIpcSupported = 143, + hostMemoryPoolsSupported = 144, + reserved145 = 145, + onlyPartialHostNativeAtomicSupported = 147, + cudaDevAttrOversizedSharedMemoryPerBlock = 150, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CUFunctionAttribute = enum(u32) { + max_threads_per_block = 0, + shared_size_bytes = 1, + const_size_bytes = 2, + local_size_bytes = 3, + num_regs = 4, + ptx_version = 5, + binary_version = 6, + cache_mode_ca = 7, + max_dynamic_shared_size_bytes = 8, + preferred_shared_memory_carveout = 9, + cluster_size_must_be_set = 10, + required_cluster_width = 11, + required_cluster_height = 12, + required_cluster_depth = 13, + non_portable_cluster_size_allowed = 14, + cluster_scheduling_policy_preference = 15, +}; + +/// Enumerated of the host library ABI. +pub const CudaABI = enum(u32) { + LP64 = 0, + ILP64 = 1, + LLP64 = 2, +}; + +/// Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members. +pub const CudaAccessProperty = enum(u32) { + cudaAccessPropertyNormal = 0, + cudaAccessPropertyStreaming = 1, + cudaAccessPropertyPersisting = 2, +}; + +/// Enumerated type describing CUDA cluster scheduling policy. +pub const CudaClusterSchedulingPolicy = enum(u32) { + cudaClusterSchedulingPolicyDefault = 0, + cudaClusterSchedulingPolicySpread = 1, + cudaClusterSchedulingPolicyLoadBalancing = 2, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CudaFuncAttribute = enum(u32) { + cudaFuncAttributeSharedMemoryMode = 16, +}; + +/// Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API. +pub const CudaLaunchAttributeID = enum(u32) { + cudaLaunchAttributeIgnore = 0, + cudaLaunchAttributeAccessPolicyWindow = 1, + cudaLaunchAttributeCooperative = 2, + cudaLaunchAttributeSynchronizationPolicy = 3, + cudaLaunchAttributeClusterDimension = 4, + cudaLaunchAttributeClusterSchedulingPolicyPreference = 5, + cudaLaunchAttributeProgrammaticStreamSerialization = 6, + cudaLaunchAttributeProgrammaticEvent = 7, + cudaLaunchAttributePriority = 8, + cudaLaunchAttributeMemSyncDomainMap = 9, + cudaLaunchAttributeMemSyncDomain = 10, + cudaLaunchAttributePreferredClusterDimension = 11, + cudaLaunchAttributeLaunchCompletionEvent = 12, + cudaLaunchAttributeDeviceUpdatableKernelNode = 13, + cudaLaunchAttributePreferredSharedMemoryCarveout = 14, + cudaLaunchAttributeNvlinkUtilCentricScheduling = 16, +}; + +/// Enumerated type describing CUDA memory sync domain. +pub const CudaLaunchMemSyncDomain = enum(u32) { + cudaLaunchMemSyncDomainDefault = 0, + cudaLaunchMemSyncDomainRemote = 1, +}; + +/// Enumerated type describing CUDA shared memory modes. +pub const CudaSharedMemoryMode = enum(u32) { + cudaSharedMemoryModeDefault = 0, + cudaSharedMemoryModeRequirePortable = 1, + cudaSharedMemoryModeAllowNonPortable = 2, + cudaSharedMemoryModeAllowOversizedSharedMemory = 3, + cudaSharedMemoryModePreferOversizedSharedMemory = 4, +}; + +/// Enumerated type describing CUDA synchronization policy. +pub const CudaSynchronizationPolicy = enum(u32) { + cudaSyncPolicyAuto = 0, + cudaSyncPolicySpin = 1, + cudaSyncPolicyYield = 2, + cudaSyncPolicyBlockingSync = 3, +}; + +/// Enumerated type describing the format of an executable. +pub const ExecutableFormat = enum(u32) { + text = 0, + elf = 1, + bitcode = 2, + fatbin = 3, +}; + +/// Enumerated type describing the representation of an executable. +pub const ExecutableRepresentation = enum(u32) { + sass = 0, + ptx = 2, + nvvm = 3, + ltoir = 4, + mlir = 5, + llvm = 6, + hostobj = 7, +}; + +/// Enumerated type denoting a GPU Architecture. +pub const GpuArchitecture = enum(u32) { + sm_80 = 80, + sm_86 = 86, + sm_87 = 87, + sm_88 = 88, + sm_89 = 89, + sm_90 = 90, + sm_100 = 100, + sm_101 = 101, + sm_103 = 103, + sm_107 = 107, + sm_110 = 110, + sm_120 = 120, + sm_121 = 121, +}; + +/// Enumerated type describing memcpy directions. +pub const MemcpyKind = enum(u32) { + HostToHost = 0, + HostToDevice = 1, + DeviceToHost = 2, + DeviceToDevice = 3, + Default = 4, +}; + +/// Enumerated type describing CUDA stream capture modes. +pub const StreamCaptureModeEnum = enum(u32) { + Global = 0, + ThreadLocal = 1, + Relaxed = 2, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel +pub const AssumeKernelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaAssumeKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "assume_kernel_attr"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaAssumeKernelAttrGet(ctx.ptr(), args.value); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) bool { + return c.mlirCudaAssumeKernelAttrGetValue(self.ptr()); + } +}; + +/// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable +pub const ComputeTargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaComputeTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "compute_target"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + representation: ExecutableRepresentation, + portability: ArchPortability, + archs: []const GpuArchitecture, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @backingInt(args.representation), @backingInt(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn representation(self: *const Self) ExecutableRepresentation { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr()))); + } + pub fn portability(self: *const Self) ArchPortability { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetPortability(self.ptr()))); + } + pub fn numArchs(self: *const Self) usize { + return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); + } + pub fn arch(self: *const Self, pos: usize) GpuArchitecture { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos)))); + } +}; + +/// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block +pub const DevMaxSharedMemoryOptinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDevMaxSharedMemoryOptin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dev_max_shared_memory_optin"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaDevMaxSharedMemoryOptinAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cuda.device_attributes`: CUDA device attributes +pub const DeviceAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDeviceAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "device_attributes"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaDeviceAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaDeviceAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "executable"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + format: ExecutableFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @backingInt(args.format)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn format_(self: *const Self) ExecutableFormat { + return @fromBackingInt(@intCast(c.mlirCudaExecutableAttrGetFormat(self.ptr()))); + } +}; + +/// `#cuda.func_attributes`: CUDA function attributes of a kernel +pub const FuncAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaFuncAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "func_attributes"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaFuncAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaFuncAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `!cuda.arch_set`: `!cuda.arch_set` +pub const ArchSetType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaArchSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "arch_set"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaArchSetTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.event`: `Event` represents CUDA event +pub const EventType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaEvent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "event"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaEventTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update +pub const GraphDeviceNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphDeviceNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_device_node"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphDeviceNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_exec`: CUDA Graph Execution +pub const GraphExecType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphExec; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_exec"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphExecTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_node`: CUDA Graph Node handle +pub const GraphNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_node"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph`: CUDA Graph +pub const GraphType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraph; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth +pub const IntType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "int"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaIntTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.kernel`: CUDA Kernel handle +pub const KernelType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "kernel"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaKernelTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API +pub const LaunchConfigType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLaunchConfig; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "launch_cfg"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + maxNumAttrs: u32, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaLaunchConfigTypeGet(ctx.ptr(), args.maxNumAttrs); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn maxNumAttrs(self: *const Self) u32 { + return c.mlirCudaLaunchConfigTypeGetMaxNumAttrs(self.ptr()); + } +}; + +/// `!cuda.library`: CUDA Library handle +pub const LibraryType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLibrary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "library"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaLibraryTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.module`: CUDA Module handle +pub const ModuleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaModule; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "module"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaModuleTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.result`: CUDA Result type +pub const ResultType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "result"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaResultTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +pub const RuntimeDevicePropType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDeviceProp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.device_prop"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDevicePropTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +pub const RuntimeDim3Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDim3; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.dim_3"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDim3TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +pub const RuntimeExtentType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeExtent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.extent"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeExtentTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). +pub const RuntimeKernelNodeParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeKernelNodeParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.kernel_node_params"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeKernelNodeParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +pub const RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cluster_scheduling_policy_preference"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` +pub const RuntimeLaunchAttributeValueCooperativeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cooperative"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` +pub const RuntimeLaunchAttributeValueMemSyncDomainType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.mem_sync_domain"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +pub const RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.nvlink_util_centric_scheduling"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +pub const RuntimeLaunchAttributeValuePortableClusterSizeModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.portable_cluster_size_mode"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` +pub const RuntimeLaunchAttributeValuePriorityType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.priority"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +pub const RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +pub const RuntimeLaunchAttributeValueSharedMemCarveoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_mem_carveout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` +pub const RuntimeLaunchAttributeValueSharedMemoryModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_memory_mode"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` +pub const RuntimeLaunchAttributeValueSyncPolicyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.sync_policy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). +pub const RuntimeMemcpy3DParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeMemcpy3DParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.memcpy_3d_params"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeMemcpy3DParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +pub const RuntimePitchedPtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePitchedPtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.pitched_ptr"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePitchedPtrTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +pub const RuntimePosType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePos; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.pos"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePosTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +pub const RuntimeUuidType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeUuid; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.uuid"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeUuidTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.stream`: CUDA Stream +pub const StreamType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaStream; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "stream"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaStreamTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.tensor_map`: CUDA Tensor Map type +pub const TensorMapType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTensorMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tensor_map"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaTensorMapTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cuda.addressof`. Result types are explicit; attributes use mlir.Attribute. +pub fn addressof(ctx: *mlir.Context, result_type: *const mlir.Type, symname: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.addressof", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "symname", symname), + }, + .location = location, + }); +} + +/// `cuda.binary`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn binary(ctx: *mlir.Context, sym_name: *const mlir.Attribute, binary_: *const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.binary", .{ + .attributes = &.{ + .named(ctx, "sym_name", sym_name), + .named(ctx, "binary", binary_), + }, + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.cast`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cuda.compose`. Result types are explicit; attributes use mlir.Attribute. +pub fn compose(ctx: *mlir.Context, fields: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.compose", .{ + .operands = .{ .flat = fields }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.decompose`. Result types are explicit; attributes use mlir.Attribute. +pub fn decompose( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + fields_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.fields_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "cuda.decompose", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = results.constSlice() }, + .location = location, + }); +} + +/// `cuda.device.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn device_sync(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.device.sync", .{ + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_create(ctx: *mlir.Context, Flags: *const mlir.Value, cuda_result_type: *const mlir.Type, phEvent_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.create", .{ + .operands = .{ .flat = &.{Flags} }, + .results = .{ .flat = &.{ cuda_result_type, phEvent_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_destroy(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.destroy", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_elapsed_time(ctx: *mlir.Context, hStart: *const mlir.Value, hEnd: *const mlir.Value, cuda_result_type: *const mlir.Type, pMilliseconds_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.elapsed_time", .{ + .operands = .{ .flat = &.{ hStart, hEnd } }, + .results = .{ .flat = &.{ cuda_result_type, pMilliseconds_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_query(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.query", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record", .{ + .operands = .{ .flat = &.{ hEvent, hStream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record_with_flags(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record_with_flags", .{ + .operands = .{ .flat = &.{ hEvent, hStream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_synchronize(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.synchronize", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_module`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_module(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, pMod_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_module", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{ cuda_result_type, pMod_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCorner: *const mlir.Value, pixelBoxUpperCorner: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col_wide`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col_wide(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCornerWidth: *const mlir.Value, pixelBoxUpperCornerWidth: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, mode: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col_wide", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.replace_address`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_replace_address(ctx: *mlir.Context, globalAddress: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.replace_address", .{ + .operands = .{ .flat = &.{globalAddress} }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_create(ctx: *mlir.Context, result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.create", .{ + .results = .{ .flat = &.{ result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_destroy(ctx: *mlir.Context, event: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.destroy", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_elapsed_time(ctx: *mlir.Context, start_event: *const mlir.Value, end_event: *const mlir.Value, cuda_result_type: *const mlir.Type, elapsed_time_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.elapsed_time", .{ + .operands = .{ .flat = &.{ start_event, end_event } }, + .results = .{ .flat = &.{ cuda_result_type, elapsed_time_type } }, + .location = location, + }); +} + +/// `cuda.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_record( + ctx: *mlir.Context, + event: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.record", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ event, value } else &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.func`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn func( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.func", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.func.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn func_get_name(ctx: *mlir.Context, kernel_: *const mlir.Value, result_type: *const mlir.Type, name_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.func.get_name", .{ + .operands = .{ .flat = &.{kernel_} }, + .results = .{ .flat = &.{ result_type, name_type } }, + .location = location, + }); +} + +/// `cuda.graph.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_destroy(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.destroy", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph.instantiate`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_instantiate(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, graph_exec_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.instantiate", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{ result_type, graph_exec_type } }, + .location = location, + }); +} + +/// `cuda.graph.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_launch(ctx: *mlir.Context, graph_exec: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.launch", .{ + .operands = .{ .flat = &.{ graph_exec, stream } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph_exec.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_exec_destroy(ctx: *mlir.Context, graph_exec: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph_exec.destroy", .{ + .operands = .{ .flat = &.{graph_exec} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.kernel`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn kernel( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + cu_func_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); + return mlir.Operation.make(ctx, "cuda.kernel", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch( + ctx: *mlir.Context, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch", .{ + .operands = .{ .variadic = &.{ + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + inputs, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_cfg.access_policy_window`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_access_policy_window(ctx: *mlir.Context, launch_cfg: *const mlir.Value, base_ptr: *const mlir.Value, hitRatio: *const mlir.Value, num_bytes: *const mlir.Value, hitProp: *const mlir.Attribute, missProp: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.access_policy_window", .{ + .operands = .{ .flat = &.{ launch_cfg, base_ptr, hitRatio, num_bytes } }, + .attributes = &.{ + .named(ctx, "hitProp", hitProp), + .named(ctx, "missProp", missProp), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, clusterDimX: *const mlir.Value, clusterDimY: *const mlir.Value, clusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, clusterDimX, clusterDimY, clusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_scheduling_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_scheduling_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, policy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_scheduling_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "policy", policy), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cooperative`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cooperative(ctx: *mlir.Context, launch_cfg: *const mlir.Value, cooperative: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cooperative", .{ + .operands = .{ .flat = &.{ launch_cfg, cooperative } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create(ctx: *mlir.Context, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ + .operands = .{ .flat = &.{ blockDimX, blockDimY, blockDimZ, dynamicSmemBytes, gridDimX, gridDimY, gridDimZ, stream } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create_from_stream`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create_from_stream(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create_from_stream", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.device_updatable_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_device_updatable_kernel_node(ctx: *mlir.Context, launch_cfg: *const mlir.Value, devNode: *const mlir.Value, deviceUpdatable: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.device_updatable_kernel_node", .{ + .operands = .{ .flat = &.{ launch_cfg, devNode, deviceUpdatable } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.grid_dimensions`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_grid_dimensions(ctx: *mlir.Context, launch_cfg: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.grid_dimensions", .{ + .operands = .{ .flat = &.{ launch_cfg, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, dynamicSmemBytes } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.launch_completion_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_launch_completion_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.launch_completion_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain(ctx: *mlir.Context, launch_cfg: *const mlir.Value, memSyncDomain: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "memSyncDomain", memSyncDomain), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain_map`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain_map(ctx: *mlir.Context, launch_cfg: *const mlir.Value, domain_default: *const mlir.Value, domain_remote: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain_map", .{ + .operands = .{ .flat = &.{ launch_cfg, domain_default, domain_remote } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.nvlink_util_centric_scheduling`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_nvlink_util_centric_scheduling(ctx: *mlir.Context, launch_cfg: *const mlir.Value, nvlinkSchedulingEnabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.nvlink_util_centric_scheduling", .{ + .operands = .{ .flat = &.{ launch_cfg, nvlinkSchedulingEnabled } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.preferred_cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_preferred_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, preferredClusterDimX: *const mlir.Value, preferredClusterDimY: *const mlir.Value, preferredClusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.preferred_cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, preferredClusterDimX, preferredClusterDimY, preferredClusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_print(ctx: *mlir.Context, launch_cfg: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.print", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .location = location, + }); +} + +/// `cuda.launch_cfg.priority`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_priority(ctx: *mlir.Context, launch_cfg: *const mlir.Value, priority: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.priority", .{ + .operands = .{ .flat = &.{ launch_cfg, priority } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, triggerAtBlockStart: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags, triggerAtBlockStart } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_stream_serialization_allowed`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, launch_cfg: *const mlir.Value, programmaticStreamSerializationAllowed: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", .{ + .operands = .{ .flat = &.{ launch_cfg, programmaticStreamSerializationAllowed } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.shared_mem_carveout`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_shared_mem_carveout(ctx: *mlir.Context, launch_cfg: *const mlir.Value, sharedMemCarveout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.shared_mem_carveout", .{ + .operands = .{ .flat = &.{ launch_cfg, sharedMemCarveout } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.sync_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_sync_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, syncPolicy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.sync_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "syncPolicy", syncPolicy), + }, + .location = location, + }); +} + +/// `cuda.launch_ex`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_ex( + ctx: *mlir.Context, + launch_cfg: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(launch_cfg); + operands.appendSliceAssumeCapacity(inputs); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch_ex", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_kernel( + ctx: *mlir.Context, + kernel_: *const mlir.Value, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + kernel_args: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.launch_kernel", .{ + .operands = .{ .variadic = &.{ + &.{kernel_}, + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + &.{kernel_args}, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, kernel_name: *const mlir.Value, result_type: *const mlir.Type, kernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, kernel_name } }, + .results = .{ .flat = &.{ result_type, kernel_type } }, + .location = location, + }); +} + +/// `cuda.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_load_data(ctx: *mlir.Context, data: *const mlir.Value, result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.load_data", .{ + .operands = .{ .flat = &.{data} }, + .results = .{ .flat = &.{ result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_unload(ctx: *mlir.Context, library: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.memory.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_async(ctx: *mlir.Context, bytes: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_async", .{ + .operands = .{ .flat = &.{ bytes, stream } }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_host(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_host", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_managed(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_managed", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy", .{ + .operands = .{ .flat = &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.copy_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy_async( + ctx: *mlir.Context, + src: *const mlir.Value, + dst: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy_async", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.free`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_async(ctx: *mlir.Context, ptr: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_async", .{ + .operands = .{ .flat = &.{ ptr, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_host(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_host", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set", .{ + .operands = .{ .flat = &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set_async( + ctx: *mlir.Context, + ptr: *const mlir.Value, + value_: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set_async", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.occupancy_max_active_blocks_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn occupancy_max_active_blocks_per_multiprocessor(ctx: *mlir.Context, block_size: *const mlir.Value, dynamic_smem_size: *const mlir.Value, res_type: *const mlir.Type, num_blocks_type: *const mlir.Type, func_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.occupancy_max_active_blocks_per_multiprocessor", .{ + .operands = .{ .flat = &.{ block_size, dynamic_smem_size } }, + .results = .{ .flat = &.{ res_type, num_blocks_type } }, + .attributes = &.{ + .named(ctx, "func", func_), + }, + .location = location, + }); +} + +/// `cuda.profiler.start`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_start(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.start", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.profiler.stop`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_stop(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.stop", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.result.assert_success`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_assert_success(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.assert_success", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.failed`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_failed(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.failed", .{ + .location = location, + }); +} + +/// `cuda.result.print_on_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_print_on_error(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.print_on_error", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.succeeded`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_succeeded(ctx: *mlir.Context, cuda_result: *const mlir.Value, success_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.succeeded", .{ + .operands = .{ .flat = &.{cuda_result} }, + .results = .{ .flat = &.{success_type} }, + .location = location, + }); +} + +/// `cuda.return`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: a terminator is only one in its block. +pub fn return_(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return", .{ + .operands = .{ .flat = inputs }, + .verify = false, + .location = location, + }); +} + +/// `cuda.return_if_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn return_if_error(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return_if_error", .{ + .operands = .{ .flat = inputs }, + .location = location, + }); +} + +/// `cuda.runtime.device.properties`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_properties(ctx: *mlir.Context, device: *const mlir.Value, cuda_result_type: *const mlir.Type, prop_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device.properties", .{ + .operands = .{ .flat = &.{device} }, + .results = .{ .flat = &.{ cuda_result_type, prop_type } }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_access_policy_max_window_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_access_policy_max_window_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_access_policy_max_window_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_async_engine_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_async_engine_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_async_engine_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_map_host_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_map_host_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_map_host_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_use_host_pointer_for_registered_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cluster_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cluster_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cluster_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_compute_preemption_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_compute_preemption_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_compute_preemption_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_kernels`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_kernels(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_kernels", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_managed_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_managed_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_managed_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cooperative_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cooperative_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cooperative_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_deferred_mapping_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_config`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_config(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_config", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_direct_managed_mem_access_from_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_direct_managed_mem_access_from_host(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ecc_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ecc_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ecc_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_global_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_global_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_global_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_flush_writes_options(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_writes_ordering(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_subsystem_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_subsystem_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_native_atomic_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_native_atomic_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_native_atomic_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_multinode_ipc_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_read_only_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_read_only_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_read_only_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_integrated`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_integrated(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_integrated", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ipc_event_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ipc_event_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ipc_event_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_is_multi_gpu_board`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_is_multi_gpu_board(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_is_multi_gpu_board", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_l_2cache_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_l_2cache_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_l_2cache_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_local_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_local_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_local_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid_device_node_mask`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid_device_node_mask(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid_device_node_mask", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_major`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_major(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_major", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_managed_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_managed_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_managed_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_blocks_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_blocks_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_grid_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_grid_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_grid_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_gather`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_gather(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_gather", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_linear`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_linear(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_linear", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d_alt`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d_alt(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d_alt", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mem_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mem_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mem_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_bus_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_bus_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_bus_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pool_supported_handle_types`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pool_supported_handle_types(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pools_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pools_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pools_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_minor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_minor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_minor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mps_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mps_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mps_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_gpu_board_group_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_gpu_board_group_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_gpu_board_group_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_processor_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_processor_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_processor_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_name(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_name", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access_uses_host_page_tables(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_bus_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_bus_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_bus_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_domain_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_domain_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_domain_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_persisting_l2cache_max_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_persisting_l2cache_max_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_persisting_l2cache_max_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_reserved_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_reserved_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block_optin`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block_optin(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block_optin", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_sparse_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_sparse_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_sparse_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_stream_priorities_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_stream_priorities_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_stream_priorities_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_surface_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_surface_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_surface_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_tcc_driver`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_tcc_driver(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_tcc_driver", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_pitch_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_pitch_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_pitch_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_timeline_semaphore_interop_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_timeline_semaphore_interop_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_const_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_const_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_const_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_global_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_global_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_global_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_addressing`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_addressing(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_addressing", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_function_pointers`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_function_pointers(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_function_pointers", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_uuid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_uuid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_uuid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_warp_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_warp_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_warp_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.create_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_create_with_flags(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.create_with_flags", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.runtime.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_query(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.query", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_record_with_flags(ctx: *mlir.Context, event: *const mlir.Value, stream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.record_with_flags", .{ + .operands = .{ .flat = &.{ event, stream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_synchronize(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.synchronize", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_depth`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_depth(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_depth", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_height`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_height(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_height", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.graph.add_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_add_kernel_node(ctx: *mlir.Context, graph: *const mlir.Value, pDependencies: *const mlir.Value, numDependencies: *const mlir.Value, pNodeParams: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraphNode_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.add_kernel_node", .{ + .operands = .{ .flat = &.{ graph, pDependencies, numDependencies, pNodeParams } }, + .results = .{ .flat = &.{ cuda_result_type, pGraphNode_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_create(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.create", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, pGraph_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_get_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_get_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, cuda_result_type: *const mlir.Type, value_out_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_get_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr } }, + .results = .{ .flat = &.{ cuda_result_type, value_out_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_set_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_set_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, value_: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_set_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr, value_ } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_block_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_block_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_block_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_extra`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_extra(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_extra", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_func`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_func(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_func", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_grid_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_grid_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_grid_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_kernel_params`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_kernel_params(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_kernel_params", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_shared_mem_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_shared_mem_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_shared_mem_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_extent`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_extent(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_extent", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_kind`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_kind(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_kind", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d(ctx: *mlir.Context, p: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d", .{ + .operands = .{ .flat = &.{p} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d_async(ctx: *mlir.Context, p: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d_async", .{ + .operands = .{ .flat = &.{ p, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_xsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_xsize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_xsize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ysize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ysize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ysize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.uuid.get_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_uuid_get_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.uuid.get_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.add_callback`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_add_callback(ctx: *mlir.Context, stream: *const mlir.Value, user_data: *const mlir.Value, result_type: *const mlir.Type, callback: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.add_callback", .{ + .operands = .{ .flat = &.{ stream, user_data } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "callback", callback), + }, + .location = location, + }); +} + +/// `cuda.stream.begin_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_begin_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.begin_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cuda.stream.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_create(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, stream_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.create", .{ + .results = .{ .flat = &.{ cuda_result_type, stream_type } }, + .location = location, + }); +} + +/// `cuda.stream.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_destroy(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.destroy", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.end_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_end_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, graph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.end_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{ result_type, graph_type } }, + .location = location, + }); +} + +/// `cuda.stream.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_synchronize( + ctx: *mlir.Context, + stream: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + event: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.synchronize", .{ + .operands = .{ .flat = if (opts.event) |value| &.{ stream, value } else &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.tensor_map_encode_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensor_map_encode_tiled(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDims: *const mlir.Value, globalStrides: *const mlir.Value, boxDims: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, cuTensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.tensor_map_encode_tiled", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDims, globalStrides, boxDims, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, cuTensorMap_type } }, + .location = location, + }); +} diff --git a/mlir/dialects/cuda/test.zig b/mlir/dialects/cuda/test.zig new file mode 100644 index 00000000..b17f50cd --- /dev/null +++ b/mlir/dialects/cuda/test.zig @@ -0,0 +1,324 @@ +const std = @import("std"); + +const cuda = @import("mlir/dialects/cuda"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda.addressof", + "cuda.binary", + "cuda.cast", + "cuda.compose", + "cuda.decompose", + "cuda.device.sync", + "cuda.driver.event.create", + "cuda.driver.event.destroy", + "cuda.driver.event.elapsed_time", + "cuda.driver.event.query", + "cuda.driver.event.record", + "cuda.driver.event.record_with_flags", + "cuda.driver.event.synchronize", + "cuda.driver.library.get_global", + "cuda.driver.library.get_kernel", + "cuda.driver.library.get_kernel_count", + "cuda.driver.library.get_managed", + "cuda.driver.library.get_module", + "cuda.driver.library.get_unified_function", + "cuda.driver.library.load_data", + "cuda.driver.library.load_from_file", + "cuda.driver.library.unload", + "cuda.driver.tensor_map.encode_im2col", + "cuda.driver.tensor_map.encode_im2col_wide", + "cuda.driver.tensor_map.replace_address", + "cuda.event.create", + "cuda.event.destroy", + "cuda.event.elapsed_time", + "cuda.event.record", + "cuda.func", + "cuda.func.get_name", + "cuda.graph.destroy", + "cuda.graph.instantiate", + "cuda.graph.launch", + "cuda.graph_exec.destroy", + "cuda.kernel", + "cuda.launch", + "cuda.launch_cfg.access_policy_window", + "cuda.launch_cfg.cluster_dim", + "cuda.launch_cfg.cluster_scheduling_policy", + "cuda.launch_cfg.cooperative", + "cuda.launch_cfg.create", + "cuda.launch_cfg.create_from_stream", + "cuda.launch_cfg.device_updatable_kernel_node", + "cuda.launch_cfg.grid_dimensions", + "cuda.launch_cfg.launch_completion_event", + "cuda.launch_cfg.mem_sync_domain", + "cuda.launch_cfg.mem_sync_domain_map", + "cuda.launch_cfg.nvlink_util_centric_scheduling", + "cuda.launch_cfg.preferred_cluster_dim", + "cuda.launch_cfg.print", + "cuda.launch_cfg.priority", + "cuda.launch_cfg.programmatic_event", + "cuda.launch_cfg.programmatic_stream_serialization_allowed", + "cuda.launch_cfg.shared_mem_carveout", + "cuda.launch_cfg.sync_policy", + "cuda.launch_ex", + "cuda.launch_kernel", + "cuda.library.get_kernel", + "cuda.library.load_data", + "cuda.library.unload", + "cuda.memory.alloc", + "cuda.memory.alloc_async", + "cuda.memory.alloc_host", + "cuda.memory.alloc_managed", + "cuda.memory.copy", + "cuda.memory.copy_async", + "cuda.memory.free", + "cuda.memory.free_async", + "cuda.memory.free_host", + "cuda.memory.set", + "cuda.memory.set_async", + "cuda.occupancy_max_active_blocks_per_multiprocessor", + "cuda.profiler.start", + "cuda.profiler.stop", + "cuda.result.assert_success", + "cuda.result.failed", + "cuda.result.print_on_error", + "cuda.result.succeeded", + "cuda.return", + "cuda.return_if_error", + "cuda.runtime.device.properties", + "cuda.runtime.device_prop.get_access_policy_max_window_size", + "cuda.runtime.device_prop.get_async_engine_count", + "cuda.runtime.device_prop.get_can_map_host_memory", + "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", + "cuda.runtime.device_prop.get_cluster_launch", + "cuda.runtime.device_prop.get_compute_preemption_supported", + "cuda.runtime.device_prop.get_concurrent_kernels", + "cuda.runtime.device_prop.get_concurrent_managed_access", + "cuda.runtime.device_prop.get_cooperative_launch", + "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", + "cuda.runtime.device_prop.get_device_numa_config", + "cuda.runtime.device_prop.get_device_numa_id", + "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", + "cuda.runtime.device_prop.get_ecc_enabled", + "cuda.runtime.device_prop.get_global_l1cache_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", + "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", + "cuda.runtime.device_prop.get_gpu_pci_device_id", + "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", + "cuda.runtime.device_prop.get_host_native_atomic_supported", + "cuda.runtime.device_prop.get_host_numa_id", + "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", + "cuda.runtime.device_prop.get_host_register_read_only_supported", + "cuda.runtime.device_prop.get_host_register_supported", + "cuda.runtime.device_prop.get_integrated", + "cuda.runtime.device_prop.get_ipc_event_supported", + "cuda.runtime.device_prop.get_is_multi_gpu_board", + "cuda.runtime.device_prop.get_l_2cache_size", + "cuda.runtime.device_prop.get_local_l1cache_supported", + "cuda.runtime.device_prop.get_luid", + "cuda.runtime.device_prop.get_luid_device_node_mask", + "cuda.runtime.device_prop.get_major", + "cuda.runtime.device_prop.get_managed_memory", + "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", + "cuda.runtime.device_prop.get_max_grid_size", + "cuda.runtime.device_prop.get_max_surface_1d", + "cuda.runtime.device_prop.get_max_surface_1d_layered", + "cuda.runtime.device_prop.get_max_surface_2d", + "cuda.runtime.device_prop.get_max_surface_2d_layered", + "cuda.runtime.device_prop.get_max_surface_3d", + "cuda.runtime.device_prop.get_max_surface_cubemap", + "cuda.runtime.device_prop.get_max_surface_cubemap_layered", + "cuda.runtime.device_prop.get_max_texture_1d", + "cuda.runtime.device_prop.get_max_texture_1d_layered", + "cuda.runtime.device_prop.get_max_texture_1d_mipmap", + "cuda.runtime.device_prop.get_max_texture_2d", + "cuda.runtime.device_prop.get_max_texture_2d_gather", + "cuda.runtime.device_prop.get_max_texture_2d_layered", + "cuda.runtime.device_prop.get_max_texture_2d_linear", + "cuda.runtime.device_prop.get_max_texture_2d_mipmap", + "cuda.runtime.device_prop.get_max_texture_3d", + "cuda.runtime.device_prop.get_max_texture_3d_alt", + "cuda.runtime.device_prop.get_max_texture_cubemap", + "cuda.runtime.device_prop.get_max_texture_cubemap_layered", + "cuda.runtime.device_prop.get_max_threads_dim", + "cuda.runtime.device_prop.get_max_threads_per_block", + "cuda.runtime.device_prop.get_max_threads_per_multi_processor", + "cuda.runtime.device_prop.get_mem_pitch", + "cuda.runtime.device_prop.get_memory_bus_width", + "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", + "cuda.runtime.device_prop.get_memory_pools_supported", + "cuda.runtime.device_prop.get_minor", + "cuda.runtime.device_prop.get_mps_enabled", + "cuda.runtime.device_prop.get_multi_gpu_board_group_id", + "cuda.runtime.device_prop.get_multi_processor_count", + "cuda.runtime.device_prop.get_name", + "cuda.runtime.device_prop.get_pageable_memory_access", + "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", + "cuda.runtime.device_prop.get_pci_bus_id", + "cuda.runtime.device_prop.get_pci_device_id", + "cuda.runtime.device_prop.get_pci_domain_id", + "cuda.runtime.device_prop.get_persisting_l2cache_max_size", + "cuda.runtime.device_prop.get_regs_per_block", + "cuda.runtime.device_prop.get_regs_per_multiprocessor", + "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block_optin", + "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", + "cuda.runtime.device_prop.get_sparse_cuda_array_supported", + "cuda.runtime.device_prop.get_stream_priorities_supported", + "cuda.runtime.device_prop.get_surface_alignment", + "cuda.runtime.device_prop.get_tcc_driver", + "cuda.runtime.device_prop.get_texture_alignment", + "cuda.runtime.device_prop.get_texture_pitch_alignment", + "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", + "cuda.runtime.device_prop.get_total_const_mem", + "cuda.runtime.device_prop.get_total_global_mem", + "cuda.runtime.device_prop.get_unified_addressing", + "cuda.runtime.device_prop.get_unified_function_pointers", + "cuda.runtime.device_prop.get_uuid", + "cuda.runtime.device_prop.get_warp_size", + "cuda.runtime.dim_3.get_x", + "cuda.runtime.dim_3.get_y", + "cuda.runtime.dim_3.get_z", + "cuda.runtime.event.create_with_flags", + "cuda.runtime.event.query", + "cuda.runtime.event.record_with_flags", + "cuda.runtime.event.synchronize", + "cuda.runtime.extent.get_depth", + "cuda.runtime.extent.get_height", + "cuda.runtime.extent.get_width", + "cuda.runtime.graph.add_kernel_node", + "cuda.runtime.graph.create", + "cuda.runtime.graph.kernel_node_get_attribute", + "cuda.runtime.graph.kernel_node_set_attribute", + "cuda.runtime.kernel_node_params.get_block_dim", + "cuda.runtime.kernel_node_params.get_extra", + "cuda.runtime.kernel_node_params.get_func", + "cuda.runtime.kernel_node_params.get_grid_dim", + "cuda.runtime.kernel_node_params.get_kernel_params", + "cuda.runtime.kernel_node_params.get_shared_mem_bytes", + "cuda.runtime.library.get_global", + "cuda.runtime.library.get_kernel", + "cuda.runtime.library.get_kernel_count", + "cuda.runtime.library.get_managed", + "cuda.runtime.library.get_unified_function", + "cuda.runtime.library.load_data", + "cuda.runtime.library.load_from_file", + "cuda.runtime.library.unload", + "cuda.runtime.memcpy_3d_params.get_dst_array", + "cuda.runtime.memcpy_3d_params.get_dst_pos", + "cuda.runtime.memcpy_3d_params.get_dst_ptr", + "cuda.runtime.memcpy_3d_params.get_extent", + "cuda.runtime.memcpy_3d_params.get_kind", + "cuda.runtime.memcpy_3d_params.get_src_array", + "cuda.runtime.memcpy_3d_params.get_src_pos", + "cuda.runtime.memcpy_3d_params.get_src_ptr", + "cuda.runtime.memory.cpy3d", + "cuda.runtime.memory.cpy3d_async", + "cuda.runtime.pitched_ptr.get_pitch", + "cuda.runtime.pitched_ptr.get_ptr", + "cuda.runtime.pitched_ptr.get_xsize", + "cuda.runtime.pitched_ptr.get_ysize", + "cuda.runtime.pos.get_x", + "cuda.runtime.pos.get_y", + "cuda.runtime.pos.get_z", + "cuda.runtime.uuid.get_bytes", + "cuda.stream.add_callback", + "cuda.stream.begin_capture", + "cuda.stream.create", + "cuda.stream.destroy", + "cuda.stream.end_capture", + "cuda.stream.synchronize", + "cuda.tensor_map_encode_tiled", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda); + std.testing.refAllDecls(cuda.ops); + inline for (comptime std.meta.declarations(cuda.ops)) |decl| { + const T = @field(cuda.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "cuda types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ cuda.ops.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, + .{ cuda.ops.ComputeTargetAttr, "#cuda.compute_target" }, + .{ cuda.ops.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, + .{ cuda.ops.ExecutableAttr, "#cuda.executable" }, + .{ cuda.ops.LaunchConfigType, "!cuda.launch_cfg" }, + .{ cuda.ops.StreamType, "!cuda.stream" }, + .{ cuda.ops.ResultType, "!cuda.result" }, + .{ cuda.ops.IntType, "!cuda.int" }, + .{ cuda.ops.RuntimeDim3Type, "!cuda.runtime.dim_3" }, + .{ cuda.ops.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, + .{ cuda.ops.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, + .{ cuda.ops.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, + .{ cuda.ops.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } + // The compiler prints these without the space before the dictionary; it + // parses both. + inline for (.{ + .{ cuda.ops.DeviceAttributesAttr, "#cuda" }, + .{ cuda.ops.FuncAttributesAttr, "#cuda" }, + }) |example| { + const parsed = try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); + } +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cuda_tile/BUILD.bazel b/mlir/dialects/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..92d622cf --- /dev/null +++ b/mlir/dialects/cuda_tile/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +# The TileView type interface, which NVIDIA's C API does not expose. +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@cuda_tile//:CudaTileDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda_tile", + import_name = "mlir/dialects/cuda_tile", + main = "cuda_tile.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@cuda_tile//:cuda_tile_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":capi", + ":cuda_tile", + "//mlir", + ], +) diff --git a/mlir/dialects/cuda_tile/capi.cc b/mlir/dialects/cuda_tile/capi.cc new file mode 100644 index 00000000..879e3757 --- /dev/null +++ b/mlir/dialects/cuda_tile/capi.cc @@ -0,0 +1,204 @@ +//===- capi.cc - C API for the cuda_tile dialect --------------------------===// + +#include "capi.h" + +#include "cuda_tile/Dialect/CudaTile/IR/Attributes.h" +#include "cuda_tile/Dialect/CudaTile/IR/Dialect.h" +#include "cuda_tile/Dialect/CudaTile/IR/Types.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" + +#include + +using namespace mlir::cuda_tile; + +static mlir::MLIRContext *loadCudaTile(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +static std::optional optional(bool has, int64_t value) { + return has ? std::optional(value) : std::nullopt; +} + +//===----------------------------------------------------------------------===// +// TileView +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileTileView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getViewTileType()); +} + +intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getViewIndexRank()); +} + +//===----------------------------------------------------------------------===// +// GatherScatterViewType +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileGatherScatterView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + auto view = llvm::dyn_cast_or_null(unwrap(tensorView)); + if (!view) + return {nullptr}; + PaddingValueAttr padding; + if (!mlirAttributeIsNull(paddingValue)) { + padding = llvm::dyn_cast(unwrap(paddingValue)); + if (!padding) + return {nullptr}; + } + auto shape = mlir::DenseI32ArrayAttr::get( + ctx, llvm::ArrayRef(tileShape, rank)); + return wrap(GatherScatterViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, shape, + view, sparseDim, padding)); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTileShape()); +} + +MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTensorView()); +} + +uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type) { + return llvm::cast(unwrap(type)).getSparseDim(); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { + return wrap( + llvm::cast(unwrap(type)).getPaddingValue()); +} + +//===----------------------------------------------------------------------===// +// OptimizationHintsAttr +//===----------------------------------------------------------------------===// + +MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, + MlirAttribute value) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + auto dict = llvm::dyn_cast_or_null(unwrap(value)); + if (!dict) + return {nullptr}; + return wrap(OptimizationHintsAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, dict)); +} + +MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValue()); +} + +//===----------------------------------------------------------------------===// +// DivByAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, + bool hasEvery, int64_t every, + bool hasAlong, int64_t along) { + mlir::MLIRContext *ctx = loadCudaTile(context); + // The printer writes `every` and `along` together. + if (!ctx || hasEvery != hasAlong) + return {nullptr}; + return wrap(DivByAttr::get(ctx, divisor, optional(hasEvery, every), + optional(hasAlong, along))); +} + +uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDivisor(); +} + +bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getEvery().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getEvery(); +} + +bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getAlong().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getAlong(); +} + +//===----------------------------------------------------------------------===// +// SameElementsAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, + intptr_t rank, + const int64_t *values) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + return wrap(SameElementsAttr::get( + ctx, mlir::DenseI64ArrayAttr::get( + ctx, llvm::ArrayRef(values, rank)))); +} + +MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValues()); +} + +//===----------------------------------------------------------------------===// +// BoundedAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileBounded(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, + int64_t lb, bool hasUb, int64_t ub) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + return wrap(BoundedAttr::get(ctx, optional(hasLb, lb), optional(hasUb, ub))); +} + +bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getLb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getLb(); +} + +bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getUb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getUb(); +} diff --git a/mlir/dialects/cuda_tile/capi.h b/mlir/dialects/cuda_tile/capi.h new file mode 100644 index 00000000..6f299c3f --- /dev/null +++ b/mlir/dialects/cuda_tile/capi.h @@ -0,0 +1,84 @@ +//===- capi.h - C API for the cuda_tile dialect -------------------*- C -*-===// +// +// C entry points for what NVIDIA's cuda_tile C API leaves out: the +// `TileView` type interface, shared by the partition, strided and +// gather/scatter views, the gather/scatter view type, the general +// `optimization_hints` constructor and the `assume` predicates. + +#ifndef MLIR_DIALECTS_CUDA_TILE_CAPI_H +#define MLIR_DIALECTS_CUDA_TILE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters; optional attribute +// parameters take null handles, and std::optional integers a `has` flag. +// Getters require a value accepted by the matching IsA. + +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileTileView(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileTileViewTypeGetViewTileType(MlirType type); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); + +// `!cuda_tile.gather_scatter_view`. `paddingValue` is a +// `#cuda_tile.padding_value` attribute, or null for none. +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileGatherScatterView(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue); +// A DenseI32ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); +// Null when the view has no padding value. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); + +// `#cuda_tile.optimization_hints` over its dictionary of per-architecture +// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGet(MlirContext context, MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); + +// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, bool hasEvery, + int64_t every, bool hasAlong, int64_t along); +MLIR_CAPI_EXPORTED uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); + +// `#cuda_tile.same_elements`. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaTileSameElements(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileSameElementsAttrGet( + MlirContext context, intptr_t rank, const int64_t *values); +// A DenseI64ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); + +// `#cuda_tile.bounded`, each bound optional. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileBoundedAttrGet( + MlirContext context, bool hasLb, int64_t lb, bool hasUb, int64_t ub); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_TILE_CAPI_H diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..ddf0a0d5 --- /dev/null +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -0,0 +1,1747 @@ +//! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one +//! function per op in `Ops.td` order. Types and attributes go through the +//! dialect's C API, completed by `capi.h` where it has no getter. + +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const dialect_namespace = "cuda_tile"; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda_tile"); +} + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn typeFromC(t: c.MlirType) *const mlir.Type { + return @ptrCast(t.ptr orelse @panic("cuda_tile: null MlirType")); +} + +fn attrFromC(a: c.MlirAttribute) *const mlir.Attribute { + return @ptrCast(a.ptr orelse @panic("cuda_tile: null MlirAttribute")); +} + +const null_attr: c.MlirAttribute = .{ .ptr = null }; + +/// `!cuda_tile.ptr` — a typed global-memory pointer. +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPointerType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const PointerType { + return @ptrCast(typeFromC(c.mlirCudaTilePointerTypeGet(ctx.ptr(), pointee_type.ptr()))); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePointerTypeGetPointeeType(self.ptr())); + } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn pointerType(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const mlir.Type { + return @ptrCast(PointerType.get(ctx, pointee_type)); +} + +/// `!cuda_tile.tile` — every SSA value that carries data. Rank 0 is +/// the scalar. Dimensions must be powers of two. +pub const TileType = opaque { + const M = mlir.Methods(TileType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATileType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: []const i64, + elem: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TileType { + return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(args.shape.len), args.shape.ptr, args.elem.ptr()))); + } + + pub fn elementType(self: *const TileType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTileTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TileType) usize { + return @intCast(c.mlirCudaTileTileTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TileType, i: usize) i64 { + return c.mlirCudaTileTileTypeGetDimSize(self.ptr(), @intCast(i)); + } + + pub fn type_(self: *const TileType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tileType(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + return @ptrCast(TileType.get(ctx, .{ .shape = shape, .elem = elem })); +} + +/// `!cuda_tile.token` — the ordering token every `*_tko` op produces. +pub const TokenType = opaque { + const M = mlir.Methods(TokenType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATokenType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context) *const TokenType { + return @ptrCast(typeFromC(c.mlirCudaTileTokenTypeGet(ctx.ptr()))); + } + + pub fn type_(self: *const TokenType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tokenType(ctx: *mlir.Context) *const mlir.Type { + return @ptrCast(TokenType.get(ctx)); +} + +/// `!cuda_tile.tensor_view`. `dynamic()` marks a +/// dimension or stride that arrives as an operand of `make_tensor_view`. +pub const TensorViewType = opaque { + const M = mlir.Methods(TensorViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATensorViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn dynamic() i64 { + return c.mlirCudaTileTensorViewTypeGetDynamicSize(); + } + + pub const InitArgs = struct { + elem: *const mlir.Type, + shape: []const i64, + strides: []const i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorViewType { + const shape = args.shape; + const strides = args.strides; + return @ptrCast(typeFromC(c.mlirCudaTileTensorViewTypeGet( + ctx.ptr(), + args.elem.ptr(), + @intCast(shape.len), + shape.ptr, + @intCast(strides.len), + strides.ptr, + ))); + } + + pub fn elementType(self: *const TensorViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTensorViewTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TensorViewType) usize { + return @intCast(c.mlirCudaTileTensorViewTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetDimSize(self.ptr(), @intCast(i)); + } + + pub fn stride(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetStride(self.ptr(), @intCast(i)); + } + + pub fn type_(self: *const TensorViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tensorViewType(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const mlir.Type { + return @ptrCast(TensorViewType.get(ctx, .{ .elem = elem, .shape = shape, .strides = strides })); +} + +const DimMap = stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK); + +fn identityDimMap(dim_map: []const i32, rank: usize) DimMap { + var out: DimMap = .empty; + if (dim_map.len > 0) { + out.appendSliceAssumeCapacity(dim_map); + } else { + for (0..rank) |i| out.appendAssumeCapacity(@intCast(i)); + } + return out; +} + +/// `!cuda_tile.partition_view, dim_map=[...]>`. +/// The tile shape, padding and dim_map all live in the type; the op that +/// builds one has no attributes of its own. +pub const PartitionViewType = opaque { + const M = mlir.Methods(PartitionViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPartitionViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// An empty `dim_map` is the identity; the type requires one entry per + /// tile dimension. + pub const InitArgs = struct { + tile_shape: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PartitionViewType { + const tile_shape = args.tile_shape; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTilePartitionViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn tensorView(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetTensorView(self.ptr())); + } + + pub fn viewTileType(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const PartitionViewType) usize { + return @intCast(c.mlirCudaTilePartitionViewTypeGetViewIndexRank(self.ptr())); + } + + pub fn type_(self: *const PartitionViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// `!cuda_tile.strided_view` (13.3). +pub const StridedViewType = opaque { + const M = mlir.Methods(StridedViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAStridedViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + tile_shape: []const i32, + traversal_strides: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const StridedViewType { + const tile_shape = args.tile_shape; + const traversal_strides = args.traversal_strides; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const strides_attr: *const mlir.Attribute = .denseArray(ctx, .i32, traversal_strides); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTileStridedViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + strides_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn viewTileType(self: *const StridedViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileStridedViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const StridedViewType) usize { + return @intCast(c.mlirCudaTileStridedViewTypeGetViewIndexRank(self.ptr())); + } + + pub fn type_(self: *const StridedViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// `!cuda_tile.gather_scatter_view, sparse_dim=N>` +/// (13.3): gathers or scatters along `sparseDim`, one tile per index. +pub const GatherScatterViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTileGatherScatterView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_view"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + tileShape: []const i32, + tensorView: *const TensorViewType, + sparseDim: u32, + paddingValue: ?PaddingValue = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileGatherScatterViewTypeGet( + ctx.ptr(), + @intCast(args.tileShape.len), + args.tileShape.ptr, + args.tensorView.ptr(), + args.sparseDim, + if (args.paddingValue) |p| p.attribute(ctx).ptr() else null_attr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn tileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { + return @ptrCast(attrFromC(c.mlirCudaTileGatherScatterViewTypeGetTileShape(self.ptr()))); + } + pub fn tensorView(self: *const Self) *const TensorViewType { + return @ptrCast(typeFromC(c.mlirCudaTileGatherScatterViewTypeGetTensorView(self.ptr()))); + } + pub fn sparseDim(self: *const Self) u32 { + return c.mlirCudaTileGatherScatterViewTypeGetSparseDim(self.ptr()); + } + pub fn paddingValue(self: *const Self) ?PaddingValue { + const attr = c.mlirCudaTileGatherScatterViewTypeGetPaddingValue(self.ptr()); + if (attr.ptr == null) return null; + const name = c.mlirCudaTilePaddingValueAttrGetValue(attr); + return std.meta.stringToEnum(PaddingValue, name.data[0..name.length]).?; + } +}; + +pub fn gatherScatterViewType( + ctx: *mlir.Context, + tile_shape: []const i64, + tensor_view: *const mlir.Type, + sparse_dim: u32, + opts: struct { + padding: ?PaddingValue = null, + }, +) *const mlir.Type { + var shape: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (tile_shape) |d| shape.appendAssumeCapacity(@intCast(d)); + const view = GatherScatterViewType.get(ctx, .{ + .tileShape = shape.constSlice(), + .tensorView = tensor_view.isA(TensorViewType) orelse std.debug.panic("cuda_tile: {f} is not a tensor_view", .{tensor_view}), + .sparseDim = sparse_dim, + .paddingValue = opts.padding, + }) catch std.debug.panic("cuda_tile: invalid gather_scatter_view over {f}", .{tensor_view}); + return view.type_(); +} + +/// The `tile<...>` type a load through `view` yields, for any view kind — +/// through the dialect's `TileView` type interface (`capi.cc`), so +/// the gather/scatter view, which has no getters in NVIDIA's C API, works too. +pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return typeFromC(c.mlirCudaTileTileViewTypeGetViewTileType(view.ptr())); +} + +pub fn indexRankOfView(view: *const mlir.Type) usize { + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return @intCast(c.mlirCudaTileTileViewTypeGetViewIndexRank(view.ptr())); +} + +// Enum attributes: built through the C getter from the tag name, so a typo is +// a null attribute rather than a wrong case. + +fn enumAttribute(comptime getter: anytype, ctx: *mlir.Context, name: []const u8) *const mlir.Attribute { + return attrFromC(getter(ctx.ptr(), stringRef(name))); +} + +pub const RoundingMode = enum { + nearest_even, + zero, + negative_inf, + positive_inf, + approx, + full, + nearest_int_to_zero, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileRoundingModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const Signedness = enum { + signed, + unsigned, + + pub fn attribute(self: Signedness, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSignednessAttrGet, ctx, @tagName(self)); + } +}; + +pub const IntegerOverflow = enum { + none, + no_signed_wrap, + no_unsigned_wrap, + no_wrap, + + pub fn attribute(self: IntegerOverflow, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileIntegerOverflowAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonOrdering = enum { + unordered, + ordered, + + pub fn attribute(self: ComparisonOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonOrderingAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonPredicate = enum { + equal, + not_equal, + less_than, + less_than_or_equal, + greater_than, + greater_than_or_equal, + + pub fn attribute(self: ComparisonPredicate, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonPredicateAttrGet, ctx, @tagName(self)); + } +}; + +pub const AtomicRMWMode = enum { + @"and", + @"or", + xor, + add, + addf, + max, + min, + umax, + umin, + xchg, + + pub fn attribute(self: AtomicRMWMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileAtomicRMWModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryScope = enum { + tl_blk, + device, + sys, + + pub fn attribute(self: MemoryScope, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryScopeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryOrdering = enum { + weak, + relaxed, + acquire, + release, + acq_rel, + + pub fn attribute(self: MemoryOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryOrderingSemanticsAttrGet, ctx, @tagName(self)); + } +}; + +pub const PaddingValue = enum { + zero, + neg_zero, + nan, + pos_inf, + neg_inf, + + pub fn attribute(self: PaddingValue, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTilePaddingValueAttrGet, ctx, @tagName(self)); + } +}; + +pub const SymbolVisibility = enum { + public, + private, + + pub fn attribute(self: SymbolVisibility, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSymbolVisibilityAttrGet, ctx, @tagName(self)); + } +}; + +// optimization_hints: one dictionary of hints per architecture. + +pub const Arch = enum { default, sm_80, sm_86, sm_87, sm_88, sm_89, sm_90, sm_100, sm_103, sm_110, sm_120, sm_121 }; + +/// Hints `entry` accepts: `num_cta_in_cga` (power of two <= 16), +/// `num_worker_warps_per_cta` (4 or 8), `occupancy`. +pub const EntryHint = struct { + arch: Arch = .default, + num_cta_in_cga: ?i32 = null, + num_worker_warps_per_cta: ?i32 = null, + occupancy: ?i32 = null, +}; + +/// Hints `load_view_tko` / `store_view_tko` accept. +pub const LoadStoreHint = struct { + arch: Arch = .default, + allow_tma: ?bool = null, + latency: ?i32 = null, +}; + +/// Hints `load_ptr_tko` / `store_ptr_tko` accept: `latency` only, the +/// pointer arm cannot use TMA. (The dialect drops unsupported keys silently.) +pub const PtrLoadStoreHint = struct { + arch: Arch = .default, + latency: ?i32 = null, +}; + +/// `#cuda_tile.optimization_hints`: a +/// dictionary from architecture name to a dictionary of hints. +pub const OptimizationHintsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirCudaTileAttributeIsAOptimizationHintsAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "optimization_hints"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: *const mlir.DictionaryAttribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileOptimizationHintsAttrGet(ctx.ptr(), args.value.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) *const mlir.DictionaryAttribute { + return @ptrCast(attrFromC(c.mlirCudaTileOptimizationHintsAttrGetValue(self.ptr()))); + } +}; + +/// The hint dictionary of one architecture: the non-null fields of `hint` +/// but `arch`, integers as `i64` like the textual form. +fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { + const field_names = @typeInfo(@TypeOf(hint)).@"struct".field_names; + var entries: stdx.BoundedArray(mlir.NamedAttribute, field_names.len) = .empty; + inline for (field_names) |field_name| { + if (comptime std.mem.eql(u8, field_name, "arch")) continue; + if (@field(hint, field_name)) |v| { + const value: *const mlir.Attribute = switch (@TypeOf(v)) { + bool => .boolean(ctx, v), + else => .int(ctx, .i64, v), + }; + entries.appendAssumeCapacity(.named(ctx, field_name, value)); + } + } + return .dict(ctx, entries.constSlice()); +} + +/// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, +/// `LoadStoreHint` or `PtrLoadStoreHint`. +pub fn optimizationHintsAttr(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { + var entries: stdx.BoundedArray(mlir.NamedAttribute, std.meta.fieldNames(Arch).len) = .empty; + for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); + const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); + const attr = OptimizationHintsAttr.get(ctx, .{ .value = value }) catch + std.debug.panic("cuda_tile: invalid optimization_hints {f}", .{value}); + return attr.attribute(); +} + +// assume predicates. + +/// `#cuda_tile.div_by`, or `` for a +/// per-group claim; `every` and `along` are both set or both null. +pub const DivByAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileDivBy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "div_by"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisor: u64, + every: ?i64 = null, + along: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileDivByAttrGet( + ctx.ptr(), + args.divisor, + args.every != null, + args.every orelse 0, + args.along != null, + args.along orelse 0, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn divisor(self: *const Self) u64 { + return c.mlirCudaTileDivByAttrGetDivisor(self.ptr()); + } + pub fn every(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasEvery(self.ptr())) c.mlirCudaTileDivByAttrGetEvery(self.ptr()) else null; + } + pub fn along(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasAlong(self.ptr())) c.mlirCudaTileDivByAttrGetAlong(self.ptr()) else null; + } +}; + +/// `along` defaults to 0 when only `every` is given. +pub fn divByAttr( + ctx: *mlir.Context, + divisor: u64, + opts: struct { + every: ?i64 = null, + along: ?i64 = null, + }, +) *const mlir.Attribute { + const attr = DivByAttr.get(ctx, .{ + .divisor = divisor, + .every = opts.every, + .along = if (opts.every != null) opts.along orelse 0 else null, + }) catch unreachable; + return attr.attribute(); +} + +/// `#cuda_tile.same_elements<[...]>`: per dimension, the size of the groups +/// of equal elements. +pub const SameElementsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileSameElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "same_elements"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileSameElementsAttrGet(ctx.ptr(), @intCast(args.values.len), args.values.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { + return @ptrCast(attrFromC(c.mlirCudaTileSameElementsAttrGetValues(self.ptr()))); + } +}; + +pub fn sameElementsAttr(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { + const attr = SameElementsAttr.get(ctx, .{ .values = values }) catch unreachable; + return attr.attribute(); +} + +/// `#cuda_tile.bounded`, either side `?` when null. +pub const BoundedAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileBounded; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bounded"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lb: ?i64 = null, + ub: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileBoundedAttrGet(ctx.ptr(), args.lb != null, args.lb orelse 0, args.ub != null, args.ub orelse 0); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn lb(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasLb(self.ptr())) c.mlirCudaTileBoundedAttrGetLb(self.ptr()) else null; + } + pub fn ub(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasUb(self.ptr())) c.mlirCudaTileBoundedAttrGetUb(self.ptr()) else null; + } +}; + +pub fn boundedAttr( + ctx: *mlir.Context, + opts: struct { + lb: ?i64 = null, + ub: ?i64 = null, + }, +) *const mlir.Attribute { + const attr = BoundedAttr.get(ctx, .{ .lb = opts.lb, .ub = opts.ub }) catch unreachable; + return attr.attribute(); +} + +// Constant values: DenseElementsAttr over a tile type, which is a ShapedType. + +pub fn splatAttr(tile_ty: *const mlir.Type, elem_attr: *const mlir.Attribute) *const mlir.Attribute { + return attrFromC(c.mlirDenseElementsAttrSplatGet(tile_ty.ptr(), elem_attr.ptr())); +} + +pub fn intElemAttr(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { + return attrFromC(c.mlirIntegerAttrGet(elem_ty.ptr(), value)); +} + +pub fn floatElemAttr(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { + return attrFromC(c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_ty.ptr(), value)); +} + +/// A dense `DenseElementsAttr` over `tile_ty` from a raw element buffer; the +/// element type's bit width must be a multiple of 8. +pub fn denseAttr(tile_ty: *const mlir.Type, values: anytype) *const mlir.Attribute { + return .denseElements(@ptrCast(tile_ty), values); +} + +const MakeArgs = mlir.Operation.MakeArgs; + +fn opName(comptime mnemonic: []const u8) []const u8 { + return dialect_namespace ++ "." ++ mnemonic; +} + +/// Attribute lists are passed by value so a helper's temporary outlives the +/// slice handed to `Operation.make`. +const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); + +fn attrsOf(list: []const mlir.NamedAttribute) Attrs { + var attrs: Attrs = .empty; + attrs.appendSliceAssumeCapacity(list); + return attrs; +} + +fn unary(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{source.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn binary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{lhs.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn convert(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, result_type: *const mlir.Type, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn roundingAttrs(ctx: *mlir.Context, rounding: RoundingMode, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", rounding.attribute(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn overflowAttrs(ctx: *mlir.Context, overflow: IntegerOverflow) Attrs { + var attrs: Attrs = .empty; + if (overflow != .none) attrs.appendAssumeCapacity(.named(ctx, "overflow", overflow.attribute(ctx))); + return attrs; +} + +fn ftzAttrs(ctx: *mlir.Context, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn i1Like(ctx: *mlir.Context, v: *const mlir.Value) *const mlir.Type { + const t = v.type_().isA(TileType) orelse std.debug.panic("cuda_tile: {f} is not a tile", .{v.type_()}); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..t.rank()) |i| shape.appendAssumeCapacity(t.dimension(i)); + return tileType(ctx, shape.constSlice(), .int(ctx, .i1)); +} + +pub fn absf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absf", source, .empty, location); +} + +pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn mulf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn divf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "divf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn remf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "remf", lhs, rhs, .empty, location); +} + +pub fn negf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negf", source, .empty, location); +} + +pub fn fma(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("fma"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = roundingAttrs(ctx, rounding, flush_to_zero).constSlice(), + .location = location, + }); +} + +fn minMaxAttrs(ctx: *mlir.Context, propagate_nan: bool, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (propagate_nan) attrs.appendAssumeCapacity(.named(ctx, "propagate_nan", .unit(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +pub fn maxf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "maxf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn minf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "minf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn pow(ctx: *mlir.Context, source: *const mlir.Value, exponent: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "fpowf", source, exponent, .empty, location); +} + +pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "atan2", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (13.3 attribute). +pub fn exp( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return unary(ctx, "exp", source, attrs, location); +} + +pub fn exp2(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "exp2", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn log(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log", source, .empty, location); +} + +pub fn log2(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log2", source, .empty, location); +} + +pub fn sqrt(ctx: *mlir.Context, source: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sqrt", source, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn rsqrt(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "rsqrt", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn sin(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sin", source, .empty, location); +} + +pub fn cos(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cos", source, .empty, location); +} + +pub fn tan(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "tan", source, .empty, location); +} + +pub fn sinh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sinh", source, .empty, location); +} + +pub fn cosh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cosh", source, .empty, location); +} + +/// `rounding` null keeps the op's default (13.2 attribute). Only `approx` +/// and `full` are legal; the upstream verifier reports the others without +/// failing, so they are refused here. +pub fn tanh( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| { + switch (r) { + .approx, .full => {}, + else => std.debug.panic("cuda_tile.tanh: rounding mode {s} is not allowed (only approx or full)", .{@tagName(r)}), + } + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + } + return unary(ctx, "tanh", source, attrs, location); +} + +pub fn ceil(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "ceil", source, .empty, location); +} + +pub fn floor(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "floor", source, .empty, location); +} + +pub fn cmpf(ctx: *mlir.Context, predicate: ComparisonPredicate, ordering: ComparisonOrdering, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpf"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "comparison_ordering", ordering.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmaf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, fast_acc: bool, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (fast_acc) attrs.appendAssumeCapacity(.named(ctx, "fast_acc", .unit(ctx))); + return mlir.Operation.make(ctx, opName("mmaf"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Block-scaled MMA (13.3): scales are plain tiles in logical M/N/K layout. +pub fn mmaf_scaled(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, lhs_scale: *const mlir.Value, rhs_scale: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmaf_scaled"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc, lhs_scale, rhs_scale } }, + .results = .{ .flat = &.{acc.type_()} }, + .location = location, + }); +} + +pub fn absi(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absi", source, .empty, location); +} + +pub fn addi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn subi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn muli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "muli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn mulhii(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulhii", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (truncation). +pub fn divi( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + signedness: Signedness, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "signedness", signedness.attribute(ctx))); + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); + return binary(ctx, "divi", lhs, rhs, attrs, location); +} + +fn signedBinary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, mnemonic, lhs, rhs, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn remi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "remi", lhs, rhs, signedness, location); +} + +pub fn maxi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "maxi", lhs, rhs, signedness, location); +} + +pub fn mini(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "mini", lhs, rhs, signedness, location); +} + +pub fn negi(ctx: *mlir.Context, source: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negi", source, overflowAttrs(ctx, overflow), location); +} + +pub fn shli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "shli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn shri(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "shri", lhs, rhs, signedness, location); +} + +pub fn cmpi(ctx: *mlir.Context, predicate: ComparisonPredicate, signedness: Signedness, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpi"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "signedness", signedness.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmai(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, signedness_lhs: Signedness, signedness_rhs: Signedness, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmai"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = &.{ + .named(ctx, "signedness_lhs", signedness_lhs.attribute(ctx)), + .named(ctx, "signedness_rhs", signedness_rhs.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn andi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "andi", lhs, rhs, .empty, location); +} + +pub fn ori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "ori", lhs, rhs, .empty, location); +} + +pub fn xori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "xori", lhs, rhs, .empty, location); +} + +pub fn bitcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "bitcast", source, result_type, .empty, location); +} + +pub fn exti(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "exti", from, to, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn trunci(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "trunci", from, to, overflowAttrs(ctx, overflow), location); +} + +/// `rounding` null keeps the op's default (nearest_even). +pub fn ftof( + ctx: *mlir.Context, + from: *const mlir.Value, + to: *const mlir.Type, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return convert(ctx, "ftof", from, to, attrs, location); +} + +pub fn ftoi(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ftoi", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn itof(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "itof", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn int_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "int_to_ptr", source, result_type, .empty, location); +} + +pub fn ptr_to_int(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_int", source, result_type, .empty, location); +} + +pub fn ptr_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_ptr", source, result_type, .empty, location); +} + +/// 13.3: sub-byte tile -> i8 tile. +pub fn pack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "pack", source, result_type, .empty, location); +} + +/// 13.3: i8 tile -> sub-byte tile. +pub fn unpack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "unpack", source, result_type, .empty, location); +} + +pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("constant"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "value", value)}, + .location = location, + }); +} + +pub fn iota(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("iota"), .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "reshape", source, result_type, .empty, location); +} + +pub fn broadcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "broadcast", source, result_type, .empty, location); +} + +pub fn cat(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, dim: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cat"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "dim", .int(ctx, .i64, dim))}, + .location = location, + }); +} + +pub fn extract(ctx: *mlir.Context, source: *const mlir.Value, indices: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, mlir.ShapedType.MAX_RANK + 1) = .empty; + operands.appendAssumeCapacity(source); + operands.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, opName("extract"), .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn permute(ctx: *mlir.Context, source: *const mlir.Value, permutation: []const i32, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "permute", source, result_type, attrsOf(&.{.named(ctx, "permutation", .denseArray(ctx, .i32, permutation))}), location); +} + +pub fn select(ctx: *mlir.Context, cond: *const mlir.Value, val_if_true: *const mlir.Value, val_if_false: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("select"), .{ + .operands = .{ .flat = &.{ cond, val_if_true, val_if_false } }, + .results = .{ .flat = &.{val_if_true.type_()} }, + .location = location, + }); +} + +pub fn offset(ctx: *mlir.Context, ptr: *const mlir.Value, off: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "offset", ptr, off, .empty, location); +} + +fn threeI32(ctx: *mlir.Context, comptime mnemonic: []const u8, location: *const mlir.Location) *mlir.Operation { + const i32_tile = tileType(ctx, &.{}, .int(ctx, .i32)); + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .results = .{ .flat = &.{ i32_tile, i32_tile, i32_tile } }, + .location = location, + }); +} + +pub fn get_tile_block_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_tile_block_id", location); +} + +pub fn get_num_tile_blocks(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_num_tile_blocks", location); +} + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?SymbolVisibility = null, +}; + +pub fn global(ctx: *mlir.Context, name: []const u8, value: *const mlir.Attribute, opts: GlobalOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, name))); + attrs.appendAssumeCapacity(.named(ctx, "value", value)); + if (opts.alignment) |a| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + if (opts.constant) attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); + if (opts.visibility) |v| attrs.appendAssumeCapacity(.named(ctx, "symbol_visibility", v.attribute(ctx))); + return mlir.Operation.make(ctx, opName("global"), .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +pub fn get_global(ctx: *mlir.Context, name: []const u8, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_global"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "name", .flatSymbolRef(ctx, name))}, + // Symbol lookup needs the enclosing module; verified with it. + .verify = false, + .location = location, + }); +} + +/// `cuda_tile.module @name { ... }`. `body` holds the entries and globals. +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("module"), .{ + .attributes = &.{.named(ctx, "sym_name", .string(ctx, name))}, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const EntryArgs = struct { + name: []const u8, + /// The entry block; its arguments are the kernel parameters, every one a + /// rank-0 tile. + block: *mlir.Block, + arg_attrs: ?[]const *const mlir.Attribute = null, + optimization_hints: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +/// `entry @name(...) { ... }` — the kernel. Returns nothing; results reach +/// the host through pointer arguments. +pub fn entry(ctx: *mlir.Context, args: EntryArgs) *mlir.Operation { + var arg_types: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (args.block.numArguments() > arg_types.capacity()) std.debug.panic("cuda_tile.entry: {d} parameters, at most {d} supported", .{ args.block.numArguments(), arg_types.capacity() }); + for (0..args.block.numArguments()) |i| arg_types.appendAssumeCapacity(args.block.argument(i).type_()); + + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); + attrs.appendAssumeCapacity(.named(ctx, "function_type", .typeAttr(.function(ctx, arg_types.constSlice(), &.{})))); + if (args.arg_attrs) |a| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, a))); + if (args.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + + return mlir.Operation.make(ctx, opName("entry"), .{ + .attributes = attrs.constSlice(), + .blocks = &.{args.block}, + .verify = false, + .location = args.location, + }); +} + +fn terminator(ctx: *mlir.Context, comptime mnemonic: []const u8, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "return", values, location); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "yield", values, location); +} + +pub fn continue_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "continue", values, location); +} + +pub fn break_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "break", values, location); +} + +/// `for %iv in (%lb to %ub, step %step) iter_values(...)`. `body` takes +/// `[iv, inits...]` and ends with `continue`. +pub fn for_(ctx: *mlir.Context, lower: *const mlir.Value, upper: *const mlir.Value, step: *const mlir.Value, inits: []const *const mlir.Value, body: *mlir.Block, unsigned_cmp: bool, location: *const mlir.Location) *mlir.Operation { + // Built on the state directly: any number of carried values, and outer + // SSA uses only dominate once the op sits in its block, so no per-op + // verify. + var state: mlir.OperationState = .init(opName("for"), location); + state.addOperands(&.{ lower, upper, step }); + state.addOperands(inits); + for (inits) |v| state.addResults(&.{v.type_()}); + if (unsigned_cmp) state.addAttributes(&.{.named(ctx, "unsignedCmp", .unit(ctx))}); + const region = mlir.Region.init(); + region.appendOwnedBlock(body); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.for"); +} + +/// `if %cond -> (types) { ... } else { ... }`. Branches end with `yield` +/// (or `continue`/`break`/`return` when they leave the enclosing scope). +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init(opName("if"), location); + state.addOperands(&.{cond}); + state.addResults(result_types); + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.if"); +} + +/// An unstructured loop over `inits`; the body takes them as arguments and +/// leaves through `continue` (next iteration) or `break` (the results). +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("loop"), .{ + .operands = .{ .flat = inits }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +/// `reduce %operands dim=D identities=[...]` — `body` takes `2N` arguments +/// (element, accumulator per operand) and yields `N` accumulators. Results +/// drop dimension `dim`. +pub fn reduce(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("reduce"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +/// `scan %operands dim=D reverse=R identities=[...]` — like `reduce`, but the +/// results keep the operand shape. +pub fn scan(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, reverse: bool, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("scan"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "reverse", .boolean(ctx, reverse)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +pub fn assert_(ctx: *mlir.Context, condition: *const mlir.Value, message: []const u8, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assert"), .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{.named(ctx, "message", .string(ctx, message))}, + .location = location, + }); +} + +/// `%r = assume #predicate, %value` — a refined copy of `value`. +pub fn assume(ctx: *mlir.Context, value: *const mlir.Value, predicate: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assume"), .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{.named(ctx, "predicate", predicate)}, + .location = location, + }); +} + +/// `make_tensor_view %base, shape = [...], strides = [...]`. Static extents +/// live in `result_type`; the operands fill its `?` slots in order. +pub fn make_tensor_view(ctx: *mlir.Context, base: *const mlir.Value, dynamic_shape: []const *const mlir.Value, dynamic_strides: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var segs: Segments = .{}; + segs.add(&.{base}); + segs.add(dynamic_shape); + segs.add(dynamic_strides); + return mlir.Operation.make(ctx, opName("make_tensor_view"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{segs.attr(ctx)}, + .location = location, + }); +} + +fn viewFromTensorView(ctx: *mlir.Context, comptime mnemonic: []const u8, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, mnemonic, tensor_view, result_type, .empty, location); +} + +/// The tile shape, padding and dim_map are all in `result_type`. +pub fn make_partition_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_partition_view", tensor_view, result_type, location); +} + +pub fn make_strided_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_strided_view", tensor_view, result_type, location); +} + +pub fn make_gather_scatter_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_gather_scatter_view", tensor_view, result_type, location); +} + +/// One rank-0 integer tile per view index dimension. +pub fn get_index_space_shape(ctx: *mlir.Context, view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_index_space_shape"), .{ + .operands = .{ .flat = &.{view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +/// One rank-0 integer tile per tensor dimension. +pub fn get_tensor_shape(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_tensor_shape"), .{ + .operands = .{ .flat = &.{tensor_view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +pub const MemOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?*const mlir.Value = null, + optimization_hints: ?*const mlir.Attribute = null, +}; + +fn memAttrs(ctx: *mlir.Context, opts: MemOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + if (opts.scope) |s| attrs.appendAssumeCapacity(.named(ctx, "memory_scope", s.attribute(ctx))); + if (opts.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + return attrs; +} + +/// An optional operand as a 0- or 1-element segment. Held in a local so the +/// slice handed to `Operation.make` stays valid. +const Seg = struct { + buf: [1]*const mlir.Value = undefined, + len: usize = 0, + + fn slice(self: *const Seg) []const *const mlir.Value { + return self.buf[0..self.len]; + } +}; + +fn optional(v: ?*const mlir.Value) Seg { + var s: Seg = .{}; + if (v) |x| { + s.buf[0] = x; + s.len = 1; + } + return s; +} + +/// Operands of an `AttrSizedOperandSegments` op, flat plus the +/// `operandSegmentSizes` the op's properties expect (a DenseI32ArrayAttr). +const Segments = struct { + values: stdx.BoundedArray(*const mlir.Value, 128) = .empty, + sizes: stdx.BoundedArray(i32, 8) = .empty, + + fn add(self: *Segments, segment: []const *const mlir.Value) void { + self.values.appendSliceAssumeCapacity(segment); + self.sizes.appendAssumeCapacity(@intCast(segment.len)); + } + + fn attr(self: *const Segments, ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, self.sizes.constSlice())); + } +}; + +/// Two results: the tile (the view's tile type) and the token. +pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + // Preserve bounds checks unless the caller has proved a coordinate in bounds. + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ tileTypeOfView(view.type_()), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// One result: the token. +pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{tile}); + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const LoadPtrOpts = struct { + mask: ?*const mlir.Value = null, + padding: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Gather through a tile of pointers. Two results: the loaded tile and the +/// token. `result_type` is the pointee tile. +pub fn load_ptr_tko(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, opts: LoadPtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const padding = optional(opts.padding); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{source}); + segs.add(mask.slice()); + segs.add(padding.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ result_type, tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StorePtrOpts = struct { + mask: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Scatter through a tile of pointers. One result: the token. +pub fn store_ptr_tko(ctx: *mlir.Context, destination: *const mlir.Value, value: *const mlir.Value, opts: StorePtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{destination}); + segs.add(&.{value}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?*const mlir.Value = null, + token: ?*const mlir.Value = null, +}; + +fn atomicAttrs(ctx: *mlir.Context, opts: AtomicOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + return attrs; +} + +/// Two results: the old values and the token. +pub fn atomic_cas_tko(ctx: *mlir.Context, pointers: *const mlir.Value, cmp: *const mlir.Value, val: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{cmp}); + segs.add(&.{val}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_cas_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ val.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Two results: the old values and the token. +pub fn atomic_rmw_tko(ctx: *mlir.Context, pointers: *const mlir.Value, mode: AtomicRMWMode, arg: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{arg}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_rmw_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ arg.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// `atomic_red_view_tko` is relaxed-only, block- or device-scoped, and has +/// no mask operand. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?*const mlir.Value = null, +}; + +/// 13.3: an atomic reduction through an unpadded tiled view. One result: +/// the token. `xchg` is not a reduction. +pub fn atomic_red_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, mode: AtomicRMWMode, value: *const mlir.Value, opts: RedViewOpts, location: *const mlir.Location) *mlir.Operation { + if (mode == .xchg) @panic("cuda_tile.atomic_red_view_tko: xchg is not a reduction"); + if (opts.scope == .sys) @panic("cuda_tile.atomic_red_view_tko: only tl_blk and device scopes are allowed"); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(&.{value}); + segs.add(tok.slice()); + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", MemoryOrdering.relaxed.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_red_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// 13.3: `num_elem` elements of the pointee, as a rank-0 pointer tile. +pub fn alloca(ctx: *mlir.Context, num_elem: i64, alignment: i64, is_global: bool, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "num_elem", .int(ctx, .i64, num_elem))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (is_global) attrs.appendAssumeCapacity(.named(ctx, "global", .unit(ctx))); + return mlir.Operation.make(ctx, opName("alloca"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn make_token(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("make_token"), .{ + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +pub fn join_tokens(ctx: *mlir.Context, tokens: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("join_tokens"), .{ + .operands = .{ .flat = tokens }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +/// Device-side printf. One result: the token. +pub fn print_tko( + ctx: *mlir.Context, + str: []const u8, + args: []const *const mlir.Value, + opts: struct { + token: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(args); + segs.add(tok.slice()); + return mlir.Operation.make(ctx, opName("print_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = &.{ .named(ctx, "str", .string(ctx, str)), segs.attr(ctx) }, + .location = location, + }); +} + +// ============================================================================= +// Tests +// ============================================================================= diff --git a/mlir/dialects/cuda_tile/test.zig b/mlir/dialects/cuda_tile/test.zig new file mode 100644 index 00000000..52ae9480 --- /dev/null +++ b/mlir/dialects/cuda_tile/test.zig @@ -0,0 +1,391 @@ +const std = @import("std"); + +const c = @import("c"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda_tile.absf", + "cuda_tile.absi", + "cuda_tile.addf", + "cuda_tile.addi", + "cuda_tile.alloca", + "cuda_tile.andi", + "cuda_tile.assert", + "cuda_tile.assume", + "cuda_tile.atan2", + "cuda_tile.atomic_cas_tko", + "cuda_tile.atomic_red_view_tko", + "cuda_tile.atomic_rmw_tko", + "cuda_tile.bitcast", + "cuda_tile.break", + "cuda_tile.broadcast", + "cuda_tile.cat", + "cuda_tile.ceil", + "cuda_tile.cmpf", + "cuda_tile.cmpi", + "cuda_tile.constant", + "cuda_tile.continue", + "cuda_tile.cos", + "cuda_tile.cosh", + "cuda_tile.divf", + "cuda_tile.divi", + "cuda_tile.entry", + "cuda_tile.exp", + "cuda_tile.exp2", + "cuda_tile.exti", + "cuda_tile.extract", + "cuda_tile.floor", + "cuda_tile.fma", + "cuda_tile.for", + "cuda_tile.fpowf", + "cuda_tile.ftof", + "cuda_tile.ftoi", + "cuda_tile.get_global", + "cuda_tile.get_index_space_shape", + "cuda_tile.get_num_tile_blocks", + "cuda_tile.get_tensor_shape", + "cuda_tile.get_tile_block_id", + "cuda_tile.global", + "cuda_tile.if", + "cuda_tile.int_to_ptr", + "cuda_tile.iota", + "cuda_tile.itof", + "cuda_tile.join_tokens", + "cuda_tile.load_ptr_tko", + "cuda_tile.load_view_tko", + "cuda_tile.log", + "cuda_tile.log2", + "cuda_tile.loop", + "cuda_tile.make_gather_scatter_view", + "cuda_tile.make_partition_view", + "cuda_tile.make_strided_view", + "cuda_tile.make_tensor_view", + "cuda_tile.make_token", + "cuda_tile.maxf", + "cuda_tile.maxi", + "cuda_tile.minf", + "cuda_tile.mini", + "cuda_tile.mmaf", + "cuda_tile.mmaf_scaled", + "cuda_tile.mmai", + "cuda_tile.module", + "cuda_tile.mulf", + "cuda_tile.mulhii", + "cuda_tile.muli", + "cuda_tile.negf", + "cuda_tile.negi", + "cuda_tile.offset", + "cuda_tile.ori", + "cuda_tile.pack", + "cuda_tile.permute", + "cuda_tile.print_tko", + "cuda_tile.ptr_to_int", + "cuda_tile.ptr_to_ptr", + "cuda_tile.reduce", + "cuda_tile.remf", + "cuda_tile.remi", + "cuda_tile.reshape", + "cuda_tile.return", + "cuda_tile.rsqrt", + "cuda_tile.scan", + "cuda_tile.select", + "cuda_tile.shli", + "cuda_tile.shri", + "cuda_tile.sin", + "cuda_tile.sinh", + "cuda_tile.sqrt", + "cuda_tile.store_ptr_tko", + "cuda_tile.store_view_tko", + "cuda_tile.subf", + "cuda_tile.subi", + "cuda_tile.tan", + "cuda_tile.tanh", + "cuda_tile.trunci", + "cuda_tile.unpack", + "cuda_tile.xori", + "cuda_tile.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda_tile); + inline for (comptime std.meta.declarations(cuda_tile)) |decl| { + const T = @field(cuda_tile, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "dialect registers and types round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); + + const f32_ty = mlir.Type.float(ctx, .f32); + const tile = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); + try std.testing.expectEqual(@as(usize, 2), tile.isA(cuda_tile.TileType).?.rank()); + try std.testing.expectEqual(@as(i64, 64), tile.isA(cuda_tile.TileType).?.dimension(1)); + + const ptr = cuda_tile.pointerType(ctx, f32_ty); + try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); + try std.testing.expect(ptr.isA(cuda_tile.PointerType).?.pointee().eql(f32_ty)); + + const dyn = cuda_tile.TensorViewType.dynamic(); + const tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); + try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); + + const static_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{16}, &.{1}); + const pv: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{2}, .tensor_view = static_tv, .padding = .zero })); + try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); + try std.testing.expect(cuda_tile.tileTypeOfView(pv).eql(cuda_tile.tileType(ctx, &.{2}, f32_ty))); + try std.testing.expectEqual(@as(usize, 1), cuda_tile.indexRankOfView(pv)); + + try std.testing.expect(cuda_tile.tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); + + const gather_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); + const gsv = cuda_tile.gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .{ .padding = .zero }); + try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); + const typed_gsv = gsv.isA(cuda_tile.GatherScatterViewType).?; + const gsv_tile = typed_gsv.tileShape(); + var gsv_shape: [2]i32 = undefined; + try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); + for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); + const rebuilt_gsv = try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &gsv_shape, + .tensorView = typed_gsv.tensorView(), + .sparseDim = typed_gsv.sparseDim(), + .paddingValue = typed_gsv.paddingValue(), + }); + try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); + try std.testing.expect(cuda_tile.tileTypeOfView(gsv).eql(cuda_tile.tileType(ctx, &.{ 8, 16 }, f32_ty))); + try std.testing.expect((try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{ 8, 16 }, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 1, + })).paddingValue() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{8}, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 0, + })); +} + +test "enum and hint attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(cuda_tile.RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); + const padding = cuda_tile.PaddingValue.neg_inf.attribute(ctx); + const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); + try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); + + const hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{ + .{ .arch = .sm_100, .num_cta_in_cga = 2 }, + .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, + }); + try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const typed_hints = hints.isA(cuda_tile.OptimizationHintsAttr).?; + try std.testing.expect((try cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.value() })).eql(typed_hints)); + const ls_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); + try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); + try std.testing.expectError(error.InvalidMlir, cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); + + const div = cuda_tile.divByAttr(ctx, 16, .{}); + try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); + const div_every = cuda_tile.divByAttr(ctx, 4, .{ .every = 4, .along = 1 }); + try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); + try std.testing.expect(cuda_tile.divByAttr(ctx, 32, .{ .every = 4 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); + const typed_div = div_every.isA(cuda_tile.DivByAttr).?; + try std.testing.expect((try cuda_tile.DivByAttr.get(ctx, .{ + .divisor = typed_div.divisor(), + .every = typed_div.every(), + .along = typed_div.along(), + })).eql(typed_div)); + try std.testing.expect(div.isA(cuda_tile.DivByAttr).?.every() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); + + const same = cuda_tile.sameElementsAttr(ctx, &.{ 2, 4 }); + try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); + const same_values = same.isA(cuda_tile.SameElementsAttr).?.values(); + try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); + try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); + + const lower = cuda_tile.boundedAttr(ctx, .{ .lb = 0 }); + try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .lb = -3, .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); + const typed_lower = lower.isA(cuda_tile.BoundedAttr).?; + try std.testing.expectEqual(@as(?i64, 0), typed_lower.lb()); + try std.testing.expect(typed_lower.ub() == null); +} + +test "entry round-trip through print and parse" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_tile = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); + + // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. + const idx_ty = cuda_tile.tileType(ctx, &.{128}, .int(ctx, .i32)); + const offsets = cuda_tile.iota(ctx, idx_ty, loc).appendTo(block); + const ptr1 = cuda_tile.tileType(ctx, &.{1}, cuda_tile.pointerType(ctx, f32_ty)); + const ptr128 = cuda_tile.tileType(ctx, &.{128}, cuda_tile.pointerType(ctx, f32_ty)); + const in_r = cuda_tile.reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); + const in_b = cuda_tile.broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); + const in_p = cuda_tile.offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); + const f32_128 = cuda_tile.tileType(ctx, &.{128}, f32_ty); + const loaded = cuda_tile.load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(f32_128, cuda_tile.floatElemAttr(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); + const sum = cuda_tile.addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); + const out_r = cuda_tile.reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); + const out_b = cuda_tile.broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); + const out_p = cuda_tile.offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); + _ = cuda_tile.store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "views, control flow and mma verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const bf16_ty = mlir.Type.float(ctx, .bf16); + const f32_ty = mlir.Type.float(ctx, .f32); + const i32_ty = mlir.Type.int(ctx, .i32); + const bf16_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, bf16_ty)); + const f32_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); + + // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. + const tv_a = cuda_tile.make_tensor_view(ctx, block.argument(0), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); + const tv_b = cuda_tile.make_tensor_view(ctx, block.argument(1), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const tv_c = cuda_tile.make_tensor_view(ctx, block.argument(2), &.{}, &.{}, cuda_tile.tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const pv_a: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_a.result(0).type_(), .padding = .zero })); + const pv_b: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_b.result(0).type_(), .padding = .zero })); + const pv_c: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_c.result(0).type_() })); + const va = cuda_tile.make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); + const vb = cuda_tile.make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); + const vc = cuda_tile.make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); + + const bid = cuda_tile.get_tile_block_id(ctx, loc).appendTo(block); + const i32_tile = cuda_tile.tileType(ctx, &.{}, i32_ty); + const zero = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 0)), i32_tile, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 1)), i32_tile, loc).appendTo(block); + const acc_ty = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + const acc0 = cuda_tile.constant(ctx, cuda_tile.splatAttr(acc_ty, cuda_tile.floatElemAttr(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); + + const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); + const at = cuda_tile.load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); + const bt = cuda_tile.load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); + try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + const acc = cuda_tile.mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); + _ = cuda_tile.continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); + const loop_op = cuda_tile.for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); + + // if %cond -> (tile) { yield } else { yield } + const cond = cuda_tile.cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); + const then_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); + const else_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); + const if_op = cuda_tile.if_(ctx, cond.result(0), &.{acc_ty}, then_block, .{ .else_block = else_block }, loc).appendTo(block); + + const store = cuda_tile.store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ + .name = "gemm", + .block = block, + .optimization_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), + .location = loc, + }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "the verifier rejects what the dialect forbids" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32_ty = mlir.Type.float(ctx, .f32); + + // An entry with a rank-1 argument. (Per-op negatives go through + // `Operation.try_make`, whose logged error the test runner counts as a + // failure, so the module-level verifier is the oracle here.) + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + const block = mlir.Block.init(&.{cuda_tile.tileType(ctx, &.{4}, f32_ty)}, &.{loc}); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + try std.testing.expect(!module_op.operation().verify()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel new file mode 100644 index 00000000..c584509f --- /dev/null +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -0,0 +1,211 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_nvgpu:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cute_ir", + import_name = "mlir/dialects/cute_ir", + main = "cute_ir.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cute_ir", + "//mlir", + "//mlir/dialects/cute_nvgpu", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CuteAttrsAll.td", + "CuteAttrsCompiler.td", + "CuteEnums.td", + "CuteOps.td", + "CuteTypesAll.td", + "CuteTypesCompiler.td", + ], + includes = ["/"], + deps = [ + "@cute_ir//:cute_ir_dialect_td", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute", + ], + "CuteAttrDecls.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute", + ], + "CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteAttrsAll.td", + deps = [":td_files"], +) + +genrule( + name = "attrs_h_inc", + srcs = ["CuteAttrDecls.h.inc"], + outs = ["CuteAttrs.h.inc"], + cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + + "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + + "echo '#endif'; cat $<) > $@", +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute", + ], + "CuteTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute", + ], + "CuteTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteTypesAll.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "CuteAttrs.h.inc", + "CuteInference.h", + ], + deps = [ + ":attrs_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + ":upstream_generated_headers", + "@cute_ir//:cute_ir_dialect_header", + "@cute_ir//:cutegen", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "CuteInference.cpp", + "CuteOps.cpp", + "CuteTypes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", + ], + deps = [ + ":headers", + "//mlir/dialects/cute_nvgpu:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) + +# Upstream CuTe sources retain their generated-header include paths. +cc_library( + name = "upstream_generated_headers", + hdrs = [ + "CuteAttrDecls.h.inc", + "CuteAttrs.cpp.inc", + "CuteAttrs.h.inc", + "CuteEnums.cpp.inc", + "CuteEnums.h.inc", + "CuteOps.cpp.inc", + "CuteOps.h.inc", + "CuteTypes.cpp.inc", + "CuteTypes.h.inc", + ], + include_prefix = "cute_ir/Dialect/Cute/IR", + strip_include_prefix = ".", +) diff --git a/mlir/dialects/cute_ir/CuteAttrsAll.td b/mlir/dialects/cute_ir/CuteAttrsAll.td new file mode 100644 index 00000000..3f42a1cb --- /dev/null +++ b/mlir/dialects/cute_ir/CuteAttrsAll.td @@ -0,0 +1,14 @@ +//===- CuteAttrsAll.td - all cute attributes ---------------*- tablegen -*-===// +// +// The dialect has one generated attribute parser and printer, so the +// release's attributes, the compiler's and its enum attributes are generated +// together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD diff --git a/mlir/dialects/cute_ir/CuteAttrsCompiler.td b/mlir/dialects/cute_ir/CuteAttrsCompiler.td new file mode 100644 index 00000000..874763a5 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteAttrsCompiler.td @@ -0,0 +1,220 @@ +// The cute dialect's attributes beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give every class, its parameters in declaration order and their +// C++ types; the live compiler, loaded from Python, gives the syntax (what it +// prints for what it parses) and the verifier messages. +// +// The fragment parameters are `FragmentAttrInterface` in the compiler, an +// interface neither the release nor this tree defines: they are plain +// attributes here (the cute_nvgpu `*_frg` attributes). +// +// The reduction_op enum attribute is in CuteEnums.td. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" + +def Cute_BitLayoutAttr : Cute_Attr<"BitLayout", "bitlayout"> { + let summary = "Bit layout of a pointer's elements within a storage chunk"; + let description = [{ + A two-mode layout from (element bit, element index) to the bit offset in + a storage chunk. Used by `!cute.ptr` (`bitlayout<"...">`). + + ```mlir + #cute.bitlayout<<"(8,4):(1,8)">> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "(element bit, element index) -> bit">:$layout + ); + let assemblyFormat = "`<` $layout `>`"; + let genVerifyDecl = 1; +} + +def Cute_CopyAtomAttr : Cute_Attr<"CopyAtom", "copy_atom"> { + let summary = "Thread and value layouts of a copy atom"; + let description = [{ + ```mlir + #cute.copy_atom, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">, <"(1,8):(0,1)">)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layout_src, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layout_dst, + Cute_Param<"LayoutAttr", "reference thread-value layout">:$layout_ref + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layout_src `,` $layout_dst `,` $layout_ref `)` `>` + }]; +} + +def Cute_CopyAtomV2Attr : Cute_Attr<"CopyAtomV2", "copy_atom_v2"> { + let summary = "Thread and value layouts and fragments of a V2 copy atom"; + let description = [{ + ```mlir + #cute.copy_atom_v2, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layoutSrc_TV, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layoutDst_TV, + Cute_Param<"mlir::Attribute", "source fragment">:$frgSrc, + Cute_Param<"mlir::Attribute", "destination fragment">:$frgDst + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layoutSrc_TV `,` $layoutDst_TV `)` `,` + `frg` `=` `(` $frgSrc `,` $frgDst `)` `>` + }]; +} + +def Cute_MmaAtomAttr : Cute_Attr<"MmaAtom", "mma_atom"> { + let summary = "Shapes, thread-value layouts and fragments of an MMA atom"; + let description = [{ + ```mlir + #cute.mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `)` `>` + }]; +} + +def Cute_MxMmaAtomAttr : Cute_Attr<"MxMmaAtom", "mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C. + + ```mlir + #cute.mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">, <"2">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `)` `>` + }]; +} + +def Cute_SparseMmaAtomAttr : Cute_Attr<"SparseMmaAtom", "sparse_mma_atom"> { + let summary = "MMA atom attribute with a sparsity-metadata operand"; + let description = [{ + Operands in the order A, B, C, E. + + ```mlir + #cute.sparse_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `,` + $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `,` $frgE `)` `>` + }]; +} + +def Cute_SparseMxMmaAtomAttr + : Cute_Attr<"SparseMxMmaAtom", "sparse_mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor and metadata operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C, E. + + ```mlir + #cute.sparse_mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"2:0">, <"3:0">, <"4:0">, <"5:0">, <"6:0">), shapes = (<"1">, <"2">, <"3">, <"4">, <"5">, <"6">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `,` $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `,` + $frgE `)` `>` + }]; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD diff --git a/mlir/dialects/cute_ir/CuteEnums.td b/mlir/dialects/cute_ir/CuteEnums.td new file mode 100644 index 00000000..68018cc1 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteEnums.td @@ -0,0 +1,70 @@ +// The cute dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD + +include "cute_ir/Dialect/Cute/IR/CuteDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cute_AddressSpaceEnum : I32EnumAttr<"AddressSpace", "Address spaces for CuTe memrefs and pointers", [ + I32EnumAttrCase<"generic", 0, "generic">, + I32EnumAttrCase<"gmem", 1, "gmem">, + I32EnumAttrCase<"smem", 3, "smem">, + I32EnumAttrCase<"rmem", 5, "rmem">, + I32EnumAttrCase<"tmem", 6, "tmem">, + I32EnumAttrCase<"dsmem", 7, "dsmem">, + I32EnumAttrCase<"cmem", 4, "cmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_CopyOperandEnum : I32EnumAttr<"CopyOperand", "Copy operand ID", [ + I32EnumAttrCase<"S", 0, "S">, + I32EnumAttrCase<"D", 1, "D"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MajorModeEnum : I32EnumAttr<"MajorMode", "Major mode for MMA operations", [ + I32EnumAttrCase<"k", 0, "k">, + I32EnumAttrCase<"mn", 1, "mn"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MmaOperandEnum : I32EnumAttr<"MmaOperand", "MMA operand ID", [ + I32EnumAttrCase<"A", 0, "A">, + I32EnumAttrCase<"B", 1, "B">, + I32EnumAttrCase<"C", 2, "C">, + I32EnumAttrCase<"D", 3, "D">, + I32EnumAttrCase<"SFA", 4, "SFA">, + I32EnumAttrCase<"SFB", 5, "SFB">, + I32EnumAttrCase<"E", 6, "E"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Op for cute reduce operations", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MUL", 1, "MUL">, + I32EnumAttrCase<"MIN", 2, "MIN">, + I32EnumAttrCase<"MAX", 3, "MAX"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD diff --git a/mlir/dialects/cute_ir/CuteInference.cpp b/mlir/dialects/cute_ir/CuteInference.cpp new file mode 100644 index 00000000..2f0415e4 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteInference.cpp @@ -0,0 +1,3600 @@ +//===- CuteInference.cpp - result types the DSL compiler would infer ------===// +// +// Every operation the DSL compiler infers result types for, inferred here +// over cutegen, so a module built in this tree carries the types the compiler +// expects. Where the compiler's rule is not recovered for some inputs (other +// atoms, tilings or layouts than those covered), inference fails rather than +// guesses. test:dialect_inference_test holds these to the compiler's choice. + +#include "mlir/dialects/cute_ir/CuteInference.h" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; +namespace cg = cutegen; +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; +// The dialect's memref, not the builtin one. +using CuteMemRefType = ::mlir::cutlass_compiler::cute::MemRefType; + +namespace { + +// Composed layouts that turned out plain are typed as plain layouts, as the +// compiler types them. +Type layoutType(MLIRContext *ctx, cg::layout l) { + return LayoutType::get(ctx, std::move(l)); +} +Type layoutType(MLIRContext *ctx, cg::composed_layout l) { + if (l.is_normal_layout()) + return LayoutType::get(ctx, l.layout_b()); + return ComposedLayoutType::get(ctx, std::move(l)); +} + +// `f` runs over the algebra value of a layout or composed layout type; the +// result keeps the input's kind (see rebuild for views). +template Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) + return {}; + return layoutType(ctx, std::move(r)); + } + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) + return {}; + return ComposedLayoutType::get(ctx, std::move(r)); + } + return {}; +} + +// What carries a layout: the layout itself, a memref (pointer + layout) or a +// coordinate tensor (arithmetic tuple + layout). +struct Carrier { + Type layout; + PtrType ptr; // memref + IntTupleType iter; // coord_tensor + Type desc; // cute_nvgpu.smem_desc_view +}; + +std::optional carrierOf(Type t) { + if (llvm::isa(t)) + return Carrier{t, {}, {}, {}}; + if (auto m = llvm::dyn_cast(t)) + return Carrier{m.getLayout(), m.getPtr(), {}, {}}; + if (auto c = llvm::dyn_cast(t)) + return Carrier{c.getLayout(), {}, c.getArithTuple(), {}}; + if (auto v = llvm::dyn_cast(t)) + return Carrier{ + LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; + return std::nullopt; +} + +Type rebuild(MLIRContext *ctx, const Carrier &c, Type layout) { + if (!layout) + return {}; + // A view's layout is plain when its composed layout turned out plain. + if (auto l = llvm::dyn_cast(layout); + l && (c.ptr || c.iter || c.desc)) + layout = layoutType(ctx, l.getRef()); + if (c.ptr) + return CuteMemRefType::get(ctx, c.ptr, layout); + if (c.iter) + return CoordTensorType::get(ctx, c.iter, layout); + if (c.desc) { + auto l = llvm::dyn_cast(layout); + return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) + : Type(); + } + return layout; +} + +// The bit width of an element type; tf32 counts as its 32-bit storage. +int64_t bitsOfType(Type t) { + if (t.isTF32()) + return 32; + return t.isIntOrFloat() ? t.getIntOrFloatBitWidth() : 0; +} + +// The alignment a pointer keeps after moving by `offset` elements: the gcd +// with what is known of the offset, a static value or a divisibility; the +// element's natural alignment when less than a byte is known. +uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { + Type elem = ptr.getValueType(); + uint64_t bits = elem && elem.isIntOrFloat() ? bitsOfType(elem) : 8; + uint64_t natural = PtrType::getNaturalAlignment(elem); + uint64_t align = ptr.getAlignment() ? ptr.getAlignment() : natural; + uint64_t known = 0; + if (cg::holds_int(offset)) { + int64_t v = offset.as_int64(); + if (v == 0) + return ptr.getAlignment(); + known = static_cast(v < 0 ? -v : v); + } else if (cg::holds_dynamic_int(offset)) { + known = static_cast( + std::get(offset).get_properties().divisibility); + } else { + return natural; + } + uint64_t result = std::gcd(align * 8, known * bits) / 8; + return result ? result : natural; +} + +PtrType movedPtr(PtrType ptr, const cg::int_tuple &offset) { + return PtrType::get(ptr.getContext(), ptr.getValueType(), + ptr.getAddressSpace(), movedAlignment(ptr, offset), + ptr.getSwizzle(), ptr.getBitlayout()); +} + +// A coordinate with its underscores read as zero, for evaluating a layout. +std::optional zeroUnderscores(const cg::coord &c) { + std::string text = cg::to_string(c); + std::replace(text.begin(), text.end(), '_', '0'); + return cg::from_string(text); +} + +// Slicing a carrier: the layout is sliced and the iterator moves by the +// layout evaluated at the coordinate. +Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { + cg::int_tuple offset; + Type layout; + if (auto l = llvm::dyn_cast(c.layout)) { + auto [sliced, off] = cg::slice_and_offset(coord, l.getRef()); + if (!cg::is_valid(sliced)) + return {}; + layout = layoutType(ctx, std::move(sliced)); + offset = off; + } else if (auto l = llvm::dyn_cast(c.layout)) { + // The composed layout's offset takes the move; the iterator stays. + auto sliced = cg::slice(coord, l.getRef()); + if (!cg::is_valid(sliced)) + return {}; + return rebuild(ctx, c, layoutType(ctx, std::move(sliced))); + } else { + return {}; + } + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); + } + // A descriptor iterator does not move. + return rebuild(ctx, c, layout); +} + +// The compiler writes a stride of 0 on every mode of extent 1 that a divide +// produces; cutegen keeps the stride the algebra gave it. +cg::layout zeroUnitModes(cg::layout l) { + if (cg::holds_leaf(l.shape())) { + if (cg::holds_int(l.shape()) && l.shape().as_int64() == 1) { + if (auto unit = cg::from_string("1:0")) + return *unit; + } + return l; + } + // Shapes and strides are congruent, so walk both spellings together. + auto shape = cg::to_string(l.shape()); + auto stride = cg::to_string(l.stride()); + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (cg::rank(fs) != cg::rank(ft)) + return l; + bool changed = false; + for (size_t i = 0; i < cg::rank(fs); ++i) { + if (cg::holds_int(fs[i]) && fs[i].as_int64() == 1 && + !(cg::holds_int(ft[i]) && ft[i].as_int64() == 0)) { + changed = true; + } + } + if (!changed) + return l; + // Rewrite the stride text leaf by leaf, in flattened order. + std::string out; + size_t leaf = 0; + auto stridesAreLeaves = [&](size_t i) { + return cg::holds_int(fs[i]) && fs[i].as_int64() == 1; + }; + size_t i = 0; + while (i < stride.size()) { + char ch = stride[i]; + if (ch == '(' || ch == ')' || ch == ',') { + out += ch; + ++i; + continue; + } + size_t j = i; + int depth = 0; + while (j < stride.size() && + !((stride[j] == ',' || stride[j] == ')') && depth == 0)) { + if (stride[j] == '<' || stride[j] == '{') + ++depth; + if (stride[j] == '>' || stride[j] == '}') + --depth; + ++j; + } + out += stridesAreLeaves(leaf) ? std::string("0") : stride.substr(i, j - i); + ++leaf; + i = j; + } + auto rebuilt = cg::from_string(shape + ":" + out); + return rebuilt ? *rebuilt : l; +} + +const cg::layout &outerOf(const cg::layout &l) { return l; } +cg::layout outerOf(const cg::composed_layout &l) { return l.layout_b(); } + +cg::composed_layout zeroUnitModes(cg::composed_layout l) { + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), + zeroUnitModes(l.layout_b())); + return cg::composed_layout(l.layout_a(), l.offset(), + zeroUnitModes(l.layout_b())); +} + +template +LogicalResult infer(std::optional loc, SmallVectorImpl &out, + F &&f) { + Type t = f(); + if (!t) + return emitOptionalError(loc, "cannot infer the result type"); + out.push_back(t); + return success(); +} + +// `like`'s nesting with its leaves replaced, in order, by `leaves`. +template +std::string nestLike(const T &like, llvm::ArrayRef leaves, + size_t &i) { + if (!cg::holds_vector(like)) + return i < leaves.size() ? leaves[i++] : "x"; + std::string s = "("; + for (size_t k = 0; k < cg::rank(like); ++k) + s += (k ? "," : "") + nestLike(like[k], leaves, i); + return s + ")"; +} + +template void leavesOf(const T &t, std::vector &out) { + if (!cg::holds_vector(t)) + return out.push_back(cg::to_string(t)); + for (size_t k = 0; k < cg::rank(t); ++k) + leavesOf(t[k], out); +} + +bool isOne(const std::string &leaf) { return leaf == "1"; } + +// The plain layout of a layout, or the outer layout of a composed one. +std::optional outerLayout(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); + return std::nullopt; +} + +// The plain layout of a layout or swizzled composed layout. +std::optional swizzledOuter(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + auto c = llvm::dyn_cast(t); + if (c && c.getRef().is_a_swizzle()) + return c.getRef().layout_b(); + return std::nullopt; +} + +// A dynamic integer's spelling: `?`, with its width when 64 and its +// divisibility when above 1. +std::string dynamicInt(int64_t div, bool wide) { + std::string props = std::string(wide ? "i64" : "") + + (wide && div > 1 ? " " : "") + + (div > 1 ? "div=" + std::to_string(div) : ""); + return props.empty() ? "?" : "?{" + props + "}"; +} + +// idx2crd of a scalar `idx` over `shape` (column-major), keeping what a +// dynamic index's divisibility tells of each coordinate, as the compiler +// does (cutegen drops it): a divisible mode's coordinate is 0; a mode whose +// extent the divisibility divides keeps it; the rest know nothing. +std::optional splitIndex(const cg::int_tuple &idx, + const cg::shape &shape) { + std::vector leaves, crd; + leavesOf(shape, leaves); + bool known = cg::holds_int(idx); + if (!known && !cg::holds_dynamic_int(idx)) + return std::nullopt; + int64_t v = known ? idx.as_int64() : 0, div = 1; + bool wide = false; + if (!known) { + const auto &p = std::get(idx).get_properties(); + div = p.divisibility; + wide = p.width == 64; + } + for (size_t i = 0; i < leaves.size(); ++i) { + int64_t n = 0; + bool staticExtent = !StringRef(leaves[i]).getAsInteger(10, n) && n > 0; + bool last = i + 1 == leaves.size(); + if (known) { + if (!staticExtent) + return std::nullopt; + crd.push_back(std::to_string(last ? v : v % n)); + v /= n; + } else if (last) { + crd.push_back(dynamicInt(div, wide)); + } else if (staticExtent && div % n == 0) { + crd.push_back("0"); + div /= n; + } else { + crd.push_back(dynamicInt(staticExtent && n % div == 0 ? div : 1, wide)); + div = 1; + } + } + size_t i = 0; + return nestLike(shape, crd, i); +} +} // namespace + +// ---------------------------------------------------------------- cute + +LogicalResult AddOffsetOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + if (!offset) + return emitOptionalError(loc, "offset must be an int tuple"); + return infer(loc, out, [&]() -> Type { + Type src = a.getSrc().getType(); + if (auto p = llvm::dyn_cast(src)) + return movedPtr(p, offset.getRef()); + if (auto it = llvm::dyn_cast(src)) { + auto sum = cg::arith_tuple_sum(it.getArithTuple().getRef(), + offset.getRef()); + if (!cg::is_valid(sum)) + return {}; + return ArithTupleIteratorType::get(ctx, IntTupleType::get(ctx, sum)); + } + return {}; + }); +} + +namespace { +// Pads `input` to `rank` modes with `element`, after (append) or before. +template +LogicalResult inferToRank(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + bool append) { + typename Op::Adaptor a(operands, attrs, props); + int32_t rank = a.getRank(); + Type in = a.getInput().getType(), e = a.getElement().getType(); + return infer(loc, out, [&]() -> Type { + if (rank <= 0) + return {}; + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + using T = decltype(ty); + // A composed layout is padded with plain layouts. + using E = std::conditional_t, + LayoutType, T>; + auto el = llvm::dyn_cast(e); + if (!el) + return {}; + auto r = append + ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) + : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); + if (!cg::is_valid(r)) + return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult AppendToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + true); +} + +LogicalResult PrependToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + false); +} + +Type mlir::cutlass_compiler::cute::assumeResultType(Type src) { + auto constrained = llvm::dyn_cast(src); + for (unsigned width : {64u, 32u}) + if (src.isInteger(width) || + (constrained && constrained.getWidth() == width)) + return ConstrainedIntType::get(src.getContext(), 1, width, false); + return {}; +} + +LogicalResult CeilDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type in = a.getInput().getType(), tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + // cutegen divides shapes; int tuples go through shape and back. + cg::shape shape; + bool tuple = false; + if (auto s = llvm::dyn_cast(in)) { + shape = s.getRef(); + } else if (auto t = llvm::dyn_cast(in)) { + shape = cg::rec_var_cast(t.getRef()); + tuple = true; + } else { + return {}; + } + cg::shape r; + if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, cg::rec_var_cast(t.getRef())); + } else { + return {}; + } + if (!cg::is_valid(r)) + return {}; + if (tuple) + return IntTupleType::get(ctx, cg::rec_var_cast(r)); + return ShapeType::get(ctx, std::move(r)); + }); +} + +LogicalResult CoalesceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + CoordType profile; + if (a.getTargetProfile()) + profile = llvm::dyn_cast(a.getTargetProfile().getType()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return profile ? cg::coalesce(l, profile.getRef()) + : cg::coalesce(l); + })); + }); +} + +LogicalResult ComposedGetOuterOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) + return {}; + return LayoutType::get(ctx, c.getRef().layout_b()); + }); +} + +LogicalResult CompositionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getLhs().getType()); + if (!c) + return emitOptionalError(loc, "lhs carries no layout"); + Type rhs = a.getRhs().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, + llvm::TypeSwitch(rhs) + .Case([&](auto r) { + return mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::composition(l, r.getRef()); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} + +LogicalResult Crd2IdxOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); + return infer(loc, out, [&]() -> Type { + auto c = zeroUnderscores(coord.getRef()); + if (!c) + return {}; + Type layout = a.getLayout().getType(); + cg::int_tuple idx; + if (auto l = llvm::dyn_cast(layout)) { + idx = cg::layout_eval(*c, l.getRef()); + } else if (auto s = llvm::dyn_cast(layout)) { + idx = cg::crd2idx(*c, s.getRef()); + } else { + return {}; + } + if (!cg::is_valid(idx)) + return {}; + return IntTupleType::get(ctx, std::move(idx)); + }); +} + +LogicalResult DereferenceArithTupleIteratorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto it = llvm::dyn_cast(operands[0].getType()); + return it ? it.getArithTuple() : Type{}; + }); +} + +LogicalResult DiceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + const cg::coord &coord = a.getCoordAttr().getRef(); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::dice(coord, l); + })); + } + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::dice(coord, ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +Type mlir::cutlass_compiler::cute::fastDivmodComputeResultType(Type divisor) { + auto d = llvm::dyn_cast(divisor); + return d ? IntegerType::get(divisor.getContext(), d.getWidth()) : Type(); +} + +Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType( + Type divisor) { + auto i = llvm::dyn_cast(divisor); + return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), + false) + : Type(); +} + +LogicalResult FilterZerosOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::filter_zeros(l); + })); + }); +} + +namespace { +// zeroUnitModes on all but the last `keep` modes (of mode 1 when `zipped`): +// the modes a divide leaves untouched keep their strides. +cg::layout zeroUnitModesHead(const cg::layout &l, size_t keep, bool zipped) { + if (!keep || !cg::holds_vector(l.shape())) + return zeroUnitModes(l); + if (zipped) { + if (cg::rank(l.shape()) != 2) + return zeroUnitModes(l); + return cg::make_layout( + std::vector{zeroUnitModes(cg::get(l, 0)), + zeroUnitModesHead(cg::get(l, 1), keep, false)}); + } + size_t n = cg::rank(l.shape()); + if (keep > n) + return l; + std::vector modes; + for (size_t i = 0; i < n; ++i) + modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) + : cg::get(l, i)); + return cg::make_layout(modes); +} +cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, + bool zipped) { + cg::layout b = zeroUnitModesHead(l.layout_b(), keep, zipped); + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), b); + return cg::composed_layout(l.layout_a(), l.offset(), b); +} + +template +LogicalResult inferDivide(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + F &÷, bool zipped = false) { + typename Op::Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + Type tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild( + ctx, *c, + llvm::TypeSwitch(tiler) + .template Case([&](auto t) { + // Unit modes the tiler passes through (`_` or extent 1) keep + // their nonzero strides, which zeroUnitModes does not model: not + // covered. + if constexpr (std::is_same_v) { + auto outer = outerLayout(c->layout); + for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && + i < cg::rank(t.getRef()) && + cg::holds_vector(outer->shape()) && + i < cg::rank(outer->shape()); + ++i) { + std::string m = cg::to_string(t.getRef()[i]); + auto tl = cg::from_string(m); + bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && + cg::size(*tl).as_int64() == 1 && + cg::to_string(tl->stride()) != "0"); + std::vector shapes, strides; + leavesOf(outer->shape()[i], shapes); + leavesOf(outer->stride()[i], strides); + for (size_t k = 0; + pass && k < shapes.size() && k < strides.size(); ++k) + if (shapes[k] == "1" && strides[k] != "0") + return Type(); + } + } + return mapLayout(ctx, c->layout, [&](const auto &l) { + // A tile or shape tiler leaves the input's trailing modes + // alone. + size_t keep = 0; + if constexpr (!std::is_same_v) { + size_t in = cg::rank(outerOf(l).shape()), + r = cg::rank(t.getRef()); + if (cg::holds_vector(outerOf(l).shape()) && r < in) + keep = in - r; + } + return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} +} // namespace + +LogicalResult FlatDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::flat_divide(l, t); }); +} + +LogicalResult LogicalDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::logical_divide(l, t); }); +} + +LogicalResult TiledDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::tiled_divide(l, t); }); +} + +LogicalResult ZippedDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, + true); +} + +Type mlir::cutlass_compiler::cute::getResultType(Type in, + ArrayRef mode) { + MLIRContext *ctx = in.getContext(); + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + auto r = cg::get(ty.getRef(), mode); + if (!cg::is_valid(r)) + return {}; + if constexpr (std::is_same_v) + return layoutType(ctx, std::move(r)); + else + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); +} + +LogicalResult GetIterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + if (auto m = llvm::dyn_cast(src)) + return m.getPtr(); + if (auto c = llvm::dyn_cast(src)) + return ArithTupleIteratorType::get(ctx, c.getArithTuple()); + if (auto v = llvm::dyn_cast(src)) + return v.getDesc(); + return {}; + }); +} + +LogicalResult GetLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = carrierOf(operands[0].getType()); + return c ? c->layout : Type{}; + }); +} + +LogicalResult GetLeavesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return llvm::TypeSwitch(operands[0].getType()) + .Case( + [&](auto ty) -> LogicalResult { + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) + out.push_back(decltype(ty)::get(ctx, flat[i])); + return success(); + }) + .Case([&](TileType ty) -> LogicalResult { + // A tile's leaves: its layouts, and `_` as a tile. + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + using T = std::decay_t; + if constexpr (cg::is_layout::value) + out.push_back(LayoutType::get(ctx, e)); + else if constexpr (!std::is_same_v>) + out.push_back(TileType::get(ctx, flat[i])); + }); + } + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the leaves of ", t); + }); +} + +LogicalResult GetScalarsOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool onlyDynamic = a.getOnlyDynamic(); + return llvm::TypeSwitch(a.getCuteValue().getType()) + .Case([&](auto ty) -> LogicalResult { + SmallVector scalars; + cg::collect_scalar_types(ctx, ty.getRef(), scalars, onlyDynamic); + out.append(scalars.begin(), scalars.end()); + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the scalars of ", t); + }); +} + +LogicalResult GetShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type in = operands[0].getType(); + if (auto t = llvm::dyn_cast(in)) + return ShapeType::get(ctx, cg::extract_shape_from_tile(t.getRef())); + auto c = carrierOf(in); + if (!c) + return {}; + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + return {}; + }); +} + +LogicalResult GroupModesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + int begin = static_cast(a.getBegin()), + end = static_cast(a.getEnd()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::group(begin, end, l); + })); + }); +} + +LogicalResult LocalTileOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tile = llvm::dyn_cast(a.getTile().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!c || !tile || !coord) + return emitOptionalError(loc, + "expects a layout carrier, a tile and a coord"); + return infer(loc, out, [&]() -> Type { + // CuTe's inner_partition: divide by the tile, take the coordinate in the + // rest modes, and flatten the two modes into one rank. + cg::tile tiler = tile.getRef(); + cg::coord crd = coord.getRef(); + if (CoordAttr proj = a.getProjAttr()) { + tiler = cg::dice(proj.getRef(), tiler); + crd = cg::dice(proj.getRef(), crd); + } + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + cg::layout zipped = cg::zipped_divide(layout.getRef(), tiler); + if (!cg::is_valid(zipped)) + return {}; + cg::layout tiles = cg::get(zipped, 0), rest = cg::get(zipped, 1); + auto [sliced, offset] = cg::slice_and_offset(crd, rest); + if (!cg::is_valid(sliced)) + return {}; + // One rank: the tile modes, then what is left of the rest modes. + std::vector modes; + for (size_t i = 0; i < cg::rank(tiles.shape()); ++i) + modes.push_back(cg::get(tiles, i)); + if (cg::holds_leaf(sliced.shape())) { + modes.push_back(sliced); + } else { + for (size_t i = 0; i < cg::rank(sliced.shape()); ++i) + modes.push_back(cg::get(sliced, i)); + } + // The input's modes beyond the tiler that the coordinate keeps are + // untouched and keep their strides. + size_t keep = 0, in = cg::rank(layout.getRef().shape()), + r = cg::rank(tiler); + for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); + ++i) + keep += cg::to_string(crd[i]) == "_"; + cg::layout flat = zeroUnitModesHead(cg::make_layout(modes), keep, false); + Type result = layoutType(ctx, std::move(flat)); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return result; + }); +} + +Type mlir::cutlass_compiler::cute::makeArithTupleIterResultType(Type tuple) { + auto t = llvm::dyn_cast(tuple); + return t ? ArithTupleIteratorType::get(tuple.getContext(), t) : Type(); +} + +LogicalResult MakeComposedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + auto outer = llvm::dyn_cast(a.getOuter().getType()); + if (!offset || !outer) + return emitOptionalError(loc, "expects an int tuple offset and a layout"); + return infer(loc, out, [&]() -> Type { + Type inner = a.getInner().getType(); + if (auto l = llvm::dyn_cast(inner)) + return ComposedLayoutType::get( + ctx, + cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); + if (auto s = llvm::dyn_cast(inner)) + return ComposedLayoutType::get( + ctx, + cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); + return {}; + }); +} + +LogicalResult MakeIdentityLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) + return {}; + cg::layout l = cg::make_identity_layout(s.getRef()); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +LogicalResult MakeOrderedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getShape().getType()); + auto order = llvm::dyn_cast(a.getOrder().getType()); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + cg::layout l(shape.getRef(), + cg::compact_order(shape.getRef(), order.getRef())); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { + MLIRContext *ctx = iter.getContext(); + if (!llvm::isa(layout)) + return {}; + if (auto p = llvm::dyn_cast(iter)) + return CuteMemRefType::get(ctx, p, layout); + if (auto it = llvm::dyn_cast(iter)) + return CoordTensorType::get(ctx, it.getArithTuple(), layout); + auto plain = llvm::dyn_cast(layout); + if (plain && llvm::isa(iter)) + return cute_nvgpu::SmemDescViewType::get(ctx, iter, plain.getAttr()); + return {}; +} + +LogicalResult +MemRefLoadOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(operands[0].getType()); + return m ? m.getValueType() : Type{}; + }); +} + +LogicalResult MemRefLoadVecOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(a.getSrc().getType()); + if (!m) + return {}; + auto l = llvm::dyn_cast(m.getLayout()); + if (!l) + return {}; + auto n = cg::size(l.getRef()); + if (!cg::holds_int(n)) + return {}; + return VectorType::get({n.as_int64()}, m.getValueType()); + }); +} + +LogicalResult SizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode; + if (auto m = a.getModeAttr()) + mode = m.asArrayRef(); + Type in = a.getInput().getType(); + if (auto c = carrierOf(in)) + in = c->layout; + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::rec_var_cast( + cg::size(cg::get(ty.getRef(), mode))); + if (!cg::is_valid(r)) + return {}; + return IntTupleType::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult SliceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) + return sliceCarrier(ctx, *c, coord.getRef()); + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::slice(coord.getRef(), ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TileToShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getTrgShape().getType()); + auto order = llvm::dyn_cast(a.getOrdShape().getType()); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + return mapLayout(ctx, a.getBlock().getType(), [&](const auto &l) { + return cg::tile_to_shape(l, shape.getRef(), order.getRef()); + }); + }); +} + +LogicalResult +ToIntTupleOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case( + [&](auto ty) -> Type { + return IntTupleType::get( + ctx, cg::rec_var_cast(ty.getRef())); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +namespace { +template +LogicalResult inferTupleArith(MLIRContext *ctx, std::optional loc, + ValueRange operands, SmallVectorImpl &out, + F &&f) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto lhs) -> Type { + using T = decltype(lhs); + auto rhs = llvm::dyn_cast(operands[1].getType()); + if (!rhs) + return {}; + auto r = f.template operator()(lhs.getRef(), + rhs.getRef()); + if (!cg::is_valid(r)) + return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult TupleAddOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sum(a, b); + }); +} + +LogicalResult TupleMulOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mul(a, b); + }); +} + +LogicalResult TupleDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_div(a, b); + }); +} + +// Without a stride the layout is compact column-major, unit modes at 0. +Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, + Type stride) { + auto s = llvm::dyn_cast(shape); + if (!s) + return {}; + MLIRContext *ctx = shape.getContext(); + if (stride) { + auto st = llvm::dyn_cast(stride); + if (!st) + return {}; + return LayoutType::get(ctx, cg::layout(s.getRef(), st.getRef())); + } + cg::layout l(s.getRef(), + cg::compact_col_major(s.getRef())); + return LayoutType::get(ctx, zeroUnitModes(l)); +} + +namespace { +// `MxNxK` and `num_cta` of an SM100 block-scaled or plain mma atom. +struct MmaAtom { + int64_t m = 0, n = 0, k = 0, ctas = 1; + Type c; // accumulator element + bool aTmem = false; // frag_kind = ts +}; +std::optional mmaAtomOf(Type tiledMma) { + auto tiled = llvm::dyn_cast(tiledMma); + if (!tiled) + return std::nullopt; + // One atom, not permuted: other tilings are not covered here. + auto n = cg::size(tiled.getAtomLayout_MNK().getRef()); + TileAttr perm = tiled.getPermutation_MNK(); + if (!cg::holds_int(n) || n.as_int64() != 1 || + (perm && cg::to_string(perm.getRef()) != "[_;_;_]")) + return std::nullopt; + MmaAtom atom; + ShapeAttr shape; + if (auto bs = llvm::dyn_cast( + tiled.getMmaAtom())) { + shape = bs.getShapeMnk(); + atom.ctas = bs.getNumCta(); + atom.c = bs.getCType(); + atom.aTmem = bs.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else if (auto mma = llvm::dyn_cast( + tiled.getMmaAtom())) { + shape = mma.getShapeMnk(); + atom.ctas = mma.getNumCta(); + atom.c = mma.getCType(); + atom.aTmem = mma.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else { + return std::nullopt; + } + if (cg::rank(shape.getRef()) != 3) + return std::nullopt; + int64_t *mnk[] = {&atom.m, &atom.n, &atom.k}; + for (int i = 0; i < 3; ++i) { + auto e = cg::get(shape.getRef(), i); + if (!cg::holds_int(e)) + return std::nullopt; + *mnk[i] = e.as_int64(); + } + return atom; +} +} // namespace + +// partition_A/B/C of the trivial tiled mma: the leading two modes are divided +// by the per-CTA atom shape, the CTA's share of the first is taken at the +// thread coordinate, and what remains follows the atom modes. +LogicalResult TiledMmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + if (!atom || !c || !coord || !which) + return emitOptionalError( + loc, "expects a tiled mma, a tensor, a coord and an operand"); + return infer(loc, out, [&]() -> Type { + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + int64_t a0, a1; + switch (which.getInt()) { + case 0: + a0 = atom->m / atom->ctas; + a1 = atom->k; + break; // A: (M, K) + case 1: + a0 = atom->n / atom->ctas; + a1 = atom->k; + break; // B: (N, K) + case 2: + a0 = atom->m / atom->ctas; + a1 = atom->n; + break; // C: (M, N) + default: + return {}; + } + const cg::layout &l = layout.getRef(); + if (cg::rank(l.shape()) < 2) + return {}; + auto tile = cg::from_string("[" + std::to_string(a0) + ":1;" + + std::to_string(a1) + ":1]"); + if (!tile) + return {}; + cg::layout two = + cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); + cg::layout divided = cg::zipped_divide(two, *tile); + if (!cg::is_valid(divided)) + return {}; + cg::layout atoms = cg::get(divided, 0), rest = cg::get(divided, 1); + // The first rest mode holds the CTA index innermost. + auto ctaTile = + cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); + if (!ctaTile) + return {}; + cg::layout rest0 = cg::zipped_divide(cg::get(rest, 0), *ctaTile); + if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) + return {}; + // A lone CTA has nothing to select: its coordinate is 0 whatever the + // thread index says. + std::string which_cta = + atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); + auto [rest0Sliced, offset] = cg::slice_and_offset( + *cg::from_string("(" + which_cta + ",_)"), rest0); + if (!cg::is_valid(rest0Sliced)) + return {}; + // What the slice leaves is one mode, not a tuple around it. + while (cg::holds_vector(rest0Sliced.shape()) && + cg::rank(rest0Sliced.shape()) == 1) + rest0Sliced = cg::get(rest0Sliced, 0); + std::vector modes{atoms, rest0Sliced, cg::get(rest, 1)}; + for (size_t i = 2; i < cg::rank(l.shape()); ++i) + modes.push_back(cg::get(l, i)); + // Modes beyond M and K (or N) are untouched. + Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), + modes.size() - 3, false)); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return result; + }); +} + +// The fragment an mma reads: for A and B a descriptor view over the staged +// smem tile, atom modes collapsed and strides in 16-byte units; for C the +// accumulator in tensor memory, one lane per row and one column per element. +LogicalResult MmaMakeFragmentOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + auto atom = mmaAtomOf(a.getAtom().getType()); + // A in tensor memory (frag_kind = ts) is not covered. + if (!which || !atom || (which.getInt() == 0 && atom->aTmem)) + return emitOptionalError(loc, + "expects an SM100 tiled mma and an operand id"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (which.getInt() == 2) { + auto shape = llvm::dyn_cast(in); + if (!shape) + return {}; + const cg::shape &s = shape.getRef(); + if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) + return {}; + auto m = cg::size(cg::get(s, 0), 0), n = cg::size(cg::get(s, 0), 1); + if (!cg::holds_int(m) || !cg::holds_int(n)) + return {}; + // ((M,N), rest...): rows on lanes, columns contiguous, stages by N + // columns; unit modes at 0. A 64-row share of a 2-CTA atom puts the + // second half of the columns on lanes 64 and up. + int64_t cols = n.as_int64(); + // Mode 0 is flattened to its (M,N) extents. + std::string text = "((" + cg::to_string(m) + "," + cg::to_string(n) + ")", + stride = "((65536,1)"; + if (atom->m / atom->ctas == 64) { + if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) + return {}; + cols /= 2; + text = "((64,(" + std::to_string(cols) + ",2))"; + stride = "((65536,(1,4194304))"; + } else if (atom->m / atom->ctas != 128) { + return {}; + } + for (size_t i = 1; i < cg::rank(s); ++i) { + auto e = cg::size(s, i); + text += "," + cg::to_string(cg::get(s, i)); + stride += "," + ((cg::holds_int(e) && e.as_int64() == 1) + ? std::string("0") + : std::to_string(cols)); + } + auto l = cg::from_string(text + "):" + stride + ")"); + if (!l) + return {}; + auto ptr = PtrType::get(ctx, atom->c, AddressSpace::tmem, 1, {}, {}); + return CuteMemRefType::get(ctx, ptr, LayoutType::get(ctx, *l)); + } + auto m = llvm::dyn_cast(in); + if (!m) + return {}; + cg::layout l; + if (auto plain = llvm::dyn_cast(m.getLayout())) + l = plain.getRef(); + else if (auto composed = llvm::dyn_cast(m.getLayout())) + l = composed.getRef().layout_b(); + else + return {}; + // The atom mode collapses to one; the rest modes follow. + int64_t bits = bitsOfType(m.getValueType()); + std::string shape = "(1", stride = "(0"; + for (size_t i = 1; i < cg::rank(l.shape()); ++i) { + auto e = cg::size(l.shape(), i); + auto st = cg::get(l.stride(), i); + if (!cg::holds_int(e) || !cg::holds_int(st)) + return {}; + shape += "," + std::to_string(e.as_int64()); + stride += "," + std::to_string(st.as_int64() * bits / 128); + } + auto view = cg::from_string(shape + "):" + stride + ")"); + if (!view) + return {}; + return cute_nvgpu::SmemDescViewType::get( + ctx, cute_nvgpu::SmemDescType::get(ctx), LayoutAttr::get(ctx, *view)); + }); +} + +LogicalResult TupleSubOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sub(a, b); + }); +} + +LogicalResult TupleModOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + if (failed(inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mod(a, b); + }))) + return failure(); + // A dynamic leaf modulo a static one: 0 when its divisibility is a + // multiple, itself when a divisor, else unknown (cutegen drops this). + std::vector lhs, rhs, res; + auto leaves = [](Type t, std::vector &l) { + if (auto i = llvm::dyn_cast(t)) + leavesOf(i.getRef(), l); + if (auto s = llvm::dyn_cast(t)) + leavesOf(s.getRef(), l); + }; + leaves(operands[0].getType(), lhs); + leaves(operands[1].getType(), rhs); + leaves(out[0], res); + if (lhs.size() != rhs.size() || lhs.size() != res.size()) + return success(); + for (size_t i = 0; i < res.size(); ++i) { + int64_t n, div = 1; + if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || + n <= 0) + continue; + size_t d = lhs[i].find("div="); + if (d != std::string::npos) + StringRef(lhs[i]) + .substr(d + 4) + .take_while(llvm::isDigit) + .getAsInteger(10, div); + res[i] = div % n == 0 ? "0" + : n % div == 0 + ? lhs[i] + : dynamicInt(1, lhs[i].find("i64") != std::string::npos); + } + size_t i = 0; + Type t = out[0]; + std::string text; + if (auto it = llvm::dyn_cast(t)) + text = nestLike(it.getRef(), res, i); + else + text = nestLike(llvm::cast(t).getRef(), res, i); + if (llvm::isa(t)) { + if (auto r = cg::from_string(text)) + out[0] = IntTupleType::get(ctx, *r); + } else if (auto r = cg::from_string(text)) { + out[0] = ShapeType::get(ctx, *r); + } + return success(); +} + +// ---------------------------------------------------------------- more cute + +namespace { + +template +LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + F &&f) { + typename Op::Adaptor a(operands, attrs, props); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + if (!tiler || !cg::is_int_or_dynamic_int_only(tiler.getRef().stride())) + return emitOptionalError(loc, "expects an integer-strided layout tiler"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + auto outer = outerLayout(in); + if (!outer || !cg::is_static(*outer) || + (!cg::is_int_or_dynamic_int_only(outer->stride()) && + !cg::is_static(tiler.getRef()))) + return {}; + return mapLayout(ctx, in, + [&](const auto &l) { return f(l, tiler.getRef()); }); + }); +} + +// A coordinate per leaf of `shape`: 0 where the extent is 1, else dynamic. +std::optional dynamicCoordLike(const cg::shape &shape) { + std::vector leaves; + leavesOf(shape, leaves); + for (auto &leaf : leaves) + leaf = isOne(leaf) ? "0" : "?"; + size_t i = 0; + return cg::from_string(nestLike(shape, leaves, i)); +} + +// Moves a carrier's iterator by `offset` and gives it `layout`. +Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, + Type layout) { + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); + } + return rebuild(ctx, c, layout); +} + +} // namespace + +#define CUTE_PRODUCT(Op, fn) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *ctx, std::optional loc, ValueRange operands, \ + DictionaryAttr attrs, PropertyRef props, RegionRange, \ + SmallVectorImpl &out) { \ + return inferProduct( \ + ctx, loc, operands, attrs, props, out, \ + [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ + } +CUTE_PRODUCT(LogicalProductOp, logical_product) +CUTE_PRODUCT(ZippedProductOp, zipped_product) +CUTE_PRODUCT(TiledProductOp, tiled_product) +CUTE_PRODUCT(FlatProductOp, flat_product) +CUTE_PRODUCT(BlockedProductOp, blocked_product) +#undef CUTE_PRODUCT + +// Unlike the other products, the raked one leaves unit modes at stride 0. +LogicalResult RakedProductOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferProduct( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { + return zeroUnitModes(cg::raked_product(l, t)); + }); +} + +// The pointer's type is kept: the compiler records no swizzle on it. +LogicalResult ApplySwizzleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +LogicalResult ComplementOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto in = llvm::dyn_cast(a.getInput().getType()); + Type co = a.getCotarget().getType(); + return infer(loc, out, [&]() -> Type { + if (!in) + return {}; + const cg::layout &l = in.getRef(); + if (!cg::is_int_or_dynamic_int_only(l.stride()) || + (!cg::is_static(l.stride()) && cg::rank(l.shape()) > 1)) + return {}; + cg::shape shape; + if (auto s = llvm::dyn_cast(co)) + shape = s.getRef(); + else if (auto c = llvm::dyn_cast(co)) + shape = c.getRef().shape(); + else + return {}; + cg::layout r = cg::complement(l, shape); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult ComposedGetInnerOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) + return {}; + if (c.getRef().is_a_swizzle()) + return SwizzleType::get(ctx, c.getRef().swizzle_a()); + return LayoutType::get(ctx, c.getRef().layout_a()); + }); +} + +LogicalResult ComposedGetOffsetOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + return c ? IntTupleType::get(ctx, c.getRef().offset()) : Type(); + }); +} + +LogicalResult CosizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + auto c = carrierOf(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + if (!c) + return {}; + cg::int_tuple r; + if (auto l = llvm::dyn_cast(c->layout)) { + r = cg::rec_var_cast( + cg::cosize(cg::get(l.getRef(), mode))); + } else { + // An affine inner layout has no cosize. + auto cl = llvm::cast(c->layout); + if (!cl.getRef().is_a_swizzle()) + return {}; + r = cg::rec_var_cast( + cg::cosize(cg::get(cl.getRef(), mode))); + } + return cg::is_valid(r) ? IntTupleType::get(ctx, std::move(r)) : Type(); + }); +} + +// The multiplier and the two shifts of the divisor's magic numbers. +LogicalResult FastDivmodGetAuxOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type i = fastDivmodComputeResultType(operands[0].getType()); + if (!i) + return emitOptionalError(loc, "expects a fast divmod divisor"); + out.append({i, IntegerType::get(ctx, 8), IntegerType::get(ctx, 8)}); + return success(); +} + +LogicalResult FastDivmodGetDivisorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&] { + return fastDivmodComputeResultType(operands[0].getType()); + }); +} + +LogicalResult FilterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + auto c = carrierOf(operands[0].getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + Type layout; + if (auto l = llvm::dyn_cast(c->layout)) { + layout = layoutType(ctx, cg::filter(l.getRef())); + } else { + // The outer layout is filtered; the inner function is kept. + const cg::composed_layout &cl = + llvm::cast(c->layout).getRef(); + cg::layout f = cg::filter(cl.layout_b()); + if (!cg::is_valid(f)) + return {}; + layout = layoutType( + ctx, cl.is_a_swizzle() + ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) + : cg::composed_layout(cl.layout_a(), cl.offset(), f)); + } + return rebuild(ctx, *c, layout); + }); +} + +namespace { +// Layout::get_hier_coord: each leaf's coordinate is (index / stride) % shape. +// A constant index gives static coordinates where the leaf is static; any +// other index gives dynamic ones, 0 on leaves of extent 1. +std::optional> hierCoord(Value index, + const cg::layout &l) { + std::vector shapes, strides; + leavesOf(l.shape(), shapes); + leavesOf(l.stride(), strides); + if (shapes.size() != strides.size()) + return std::nullopt; + APInt value; + bool constant = matchPattern(index, m_ConstantInt(&value)); + std::vector crd; + for (size_t i = 0; i < shapes.size(); ++i) { + int64_t n = 0, d = 0; + bool staticShape = !StringRef(shapes[i]).getAsInteger(10, n); + if (staticShape && n == 1) { + crd.push_back("0"); + } else if (!constant) { + crd.push_back("?"); + } else if (StringRef(strides[i]).getAsInteger(10, d) || d <= 0) { + return std::nullopt; // not recovered for dynamic or zero strides + } else { + int64_t q = value.getSExtValue() / d; + crd.push_back(staticShape ? std::to_string(q % n) : "?"); + } + } + return crd; +} + +// Column-major index of static coordinates in `shape`, or "?". +std::string colMajor(const std::vector &crd, + const std::vector &shapes, size_t begin, + size_t end) { + int64_t idx = 0, scale = 1; + for (size_t i = begin; i < end; ++i) { + int64_t c, n; + if (StringRef(crd[i]).getAsInteger(10, c)) + return "?"; + idx += c * scale; + if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) + return "?"; + scale *= n; + } + return std::to_string(idx); +} +} // namespace + +LogicalResult GetHierCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + size_t i = 0; + auto c = + cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +// One coordinate per mode: the mode's hierarchical coordinate, column-major. +LogicalResult GetFlatCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + const cg::shape &shape = layout.getRef().shape(); + std::vector shapes, modes; + leavesOf(shape, shapes); + if (!cg::holds_vector(shape)) { + modes.push_back(colMajor(*crd, shapes, 0, shapes.size())); + } else { + size_t begin = 0; + for (size_t m = 0; m < cg::rank(shape); ++m) { + std::vector sub; + leavesOf(shape[m], sub); + modes.push_back(colMajor(*crd, shapes, begin, begin + sub.size())); + begin += sub.size(); + } + } + auto c = cg::from_string(cg::holds_vector(shape) + ? "(" + llvm::join(modes, ",") + ")" + : modes[0]); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetIntegralCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + std::vector shapes; + leavesOf(layout.getRef().shape(), shapes); + auto c = + cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetLayoutsFromTileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto tile = llvm::dyn_cast(operands[0].getType()); + if (!tile) + return emitOptionalError(loc, "expects a tile"); + auto flat = cg::flatten(tile.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + if constexpr (cg::is_layout>::value) + out.push_back(LayoutType::get(ctx, e)); + }); + } + if (out.empty()) + return emitOptionalError(loc, "the tile holds no layout"); + return success(); +} + +LogicalResult GetStrideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(operands[0].getType()); + return l ? StrideType::get(ctx, l.getRef().stride()) : Type(); + }); +} + +LogicalResult Idx2CrdOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto index = llvm::dyn_cast(a.getIndex().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!index || !shape) + return {}; + size_t r = cg::rank(index.getRef()); + if (r > 1 && r != cg::rank(shape.getRef())) + return {}; + if (auto split = cg::holds_dynamic_int(index.getRef()) + ? splitIndex(index.getRef(), shape.getRef()) + : std::nullopt) { + auto c = cg::from_string(*split); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + } + auto c = cg::rec_var_cast( + cg::idx2crd(index.getRef(), shape.getRef())); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +LogicalResult IncrementCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!coord || !shape || !cg::is_congruent(coord.getRef(), shape.getRef()) || + cg::has_underscore(coord.getRef())) + return {}; + auto c = cg::increment_coord(coord.getRef(), shape.getRef()); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +namespace { +template +LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, + Type input, bool staticStride, + SmallVectorImpl &out, F &&f) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(input); + if (!l || !cg::is_static(l.getRef().shape()) || + (staticStride && !cg::is_static(l.getRef().stride()))) + return {}; + cg::layout r = f(l.getRef()); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} +} // namespace + +LogicalResult LeftInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), true, out, + [](const cg::layout &l) { return cg::left_inverse(l); }); +} + +LogicalResult RightInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), false, out, + [](const cg::layout &l) { return cg::right_inverse(l); }); +} + +LogicalResult LoadScaledIndexOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// CuTe's local_partition: the tensor divided by the tiler's shape, sliced at +// the thread's (dynamic) coordinate in the tile. +LogicalResult LocalPartitionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + return infer(loc, out, [&]() -> Type { + auto layout = c ? llvm::dyn_cast(c->layout) : LayoutType(); + if (!layout || !tiler || !cg::is_static(tiler.getRef())) + return {}; + cg::shape tile = cg::product_each(tiler.getRef().shape()); + cg::layout zipped = cg::zipped_divide(layout.getRef(), tile); + auto thread = dynamicCoordLike(tile); + if (!cg::is_valid(zipped) || !thread) + return {}; + auto crd = cg::from_string("(" + cg::to_string(*thread) + ",_)"); + if (!crd) + return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, zipped); + if (!cg::is_valid(sliced)) + return {}; + // The slice keeps the rest mode's own modes, not a tuple around them. + if (cg::holds_vector(sliced.shape()) && cg::rank(sliced.shape()) == 1) + sliced = cg::get(sliced, 0); + return moved(ctx, *c, offset, LayoutType::get(ctx, zeroUnitModes(sliced))); + }); +} + +// CuTe's make_fragment_like: the first mode compact column-major with its +// stride-0 modes kept, the other modes ordered by their strides (dynamic +// ones last) after it; column-major when there is one mode or the first +// mode's shape is dynamic. +LogicalResult MakeFragmentLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto src = swizzledOuter(operands[0].getType()); + return infer(loc, out, [&]() -> Type { + if (!src) + return {}; + const cg::shape &shape = src->shape(); + std::vector shapes, strides; + leavesOf(shape, shapes); + leavesOf(src->stride(), strides); + if (shapes.size() != strides.size()) + return {}; + size_t first = shapes.size(); // leaves of mode 0 + bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && + cg::is_static(shape[0]); + if (ordered) { + std::vector m0; + leavesOf(shape[0], m0); + first = m0.size(); + } + // The order the leaves are laid out in. + std::vector order(shapes.size()); + std::iota(order.begin(), order.end(), 0); + auto key = [&](size_t i) -> int64_t { + int64_t v; + return StringRef(strides[i]).getAsInteger(10, v) ? INT64_MAX : v; + }; + if (ordered) + std::stable_sort(order.begin() + first, order.end(), + [&](size_t x, size_t y) { return key(x) < key(y); }); + std::vector result(shapes.size()); + cg::int_tuple step = *cg::from_string("1"); + for (size_t i : order) { + if (isOne(shapes[i]) || (ordered && i < first && strides[i] == "0")) { + result[i] = "0"; + continue; + } + result[i] = cg::to_string(step); + auto n = cg::from_string(shapes[i]); + if (!n) + return {}; + step = cg::arith_tuple_mul(step, *n); + } + size_t i = 0; + auto l = cg::from_string(cg::to_string(shape) + ":" + + nestLike(shape, result, i)); + return l ? LayoutType::get(ctx, std::move(*l)) : Type(); + }); +} + +LogicalResult MakeIdentityTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) + return {}; + std::vector leaves; + leavesOf(s.getRef(), leaves); + for (auto &leaf : leaves) + leaf = "0"; + size_t i = 0; + auto zero = cg::from_string(nestLike(s.getRef(), leaves, i)); + cg::layout l = cg::make_identity_layout(s.getRef()); + if (!zero || !cg::is_valid(l)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), + LayoutType::get(ctx, l)); + }); +} + +LogicalResult MakeLayoutLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + cg::layout r; + if (auto l = llvm::dyn_cast(src)) { + if (cg::has_scaled_basis(l.getRef().stride())) + return {}; + r = cg::make_layout_like(l.getRef()); + } else if (auto c = llvm::dyn_cast(src)) { + if (!c.getRef().is_a_swizzle() || + cg::has_scaled_basis(c.getRef().layout_b().stride())) + return {}; + r = cg::make_layout_like(c.getRef()); + } else { + return {}; + } + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult PtrLoadOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto p = llvm::dyn_cast(operands[0].getType()); + return p ? p.getValueType() : Type(); + }); +} + +LogicalResult RecastLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + int newBits = a.getNewTypeBits(), oldBits = a.getOldTypeBits(); + return infer(loc, out, [&]() -> Type { + if (newBits <= 0 || oldBits <= 0) + return {}; + auto outer = outerLayout(a.getSrc().getType()); + if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) + return {}; + return mapLayout(ctx, a.getSrc().getType(), [&](const auto &l) { + return cg::recast(newBits, oldBits, l); + }); + }); +} + +// Without a profile the whole vector reduces to a scalar; with one, the +// modes it marks reduce and the others (`_`) remain, as a vector and shape. +LogicalResult ReduceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto vec = llvm::dyn_cast(a.getInput().getType()); + if (!vec) + return emitOptionalError(loc, "expects a vector input"); + if (!a.getReductionProfileAttr()) { + out.push_back(vec.getElementType()); + return success(); + } + auto shape = llvm::dyn_cast(a.getShape().getType()); + if (!shape) + return emitOptionalError(loc, "expects a shape"); + cg::shape rest = + cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); + auto n = cg::size(rest); + if (!cg::is_valid(rest) || !cg::holds_int(n)) + return emitOptionalError(loc, "cannot reduce the shape"); + out.push_back(VectorType::get({n.as_int64()}, vec.getElementType())); + out.push_back(ShapeType::get(ctx, std::move(rest))); + return success(); +} + +LogicalResult SelectOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(a.getInput().getType()) + .Case([&](auto ty) -> Type { + size_t r = cg::rank(ty.getRef()); + for (int32_t m : mode) + if (m < 0 || static_cast(m) >= r) + return {}; + auto res = cg::select(ty.getRef(), mode); + if (!cg::is_valid(res)) + return {}; + return decltype(ty)::get(ctx, std::move(res)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult ShapeDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto x = llvm::dyn_cast(operands[0].getType()); + auto y = llvm::dyn_cast(operands[1].getType()); + if (!x || !y) + return {}; + cg::shape r = cg::shape_div(x.getRef(), y.getRef()); + return cg::is_valid(r) ? ShapeType::get(ctx, std::move(r)) : Type(); + }); +} + +// The compiler aborts on every input tried (bad_alloc, segfault): no rule. +LogicalResult StencilDivideOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &) { + return emitOptionalError(loc, + "cute.stencil_divide inference is not recovered"); +} + +// No type of the compiler's registry implements the descriptor iterator +// interface this needs, so the compiler rejects every input. +LogicalResult DereferenceDescriptorIteratorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return emitOptionalError(loc, "expects a descriptor iterator"); +} + +LogicalResult TupleProductOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TupleProductEachOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product_each(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +// A SIMT copy's fragment is the tensor itself; the rule for other atoms is +// not recovered. +LogicalResult +CopyMakeFragmentOp::inferReturnTypes(MLIRContext *, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tiled = llvm::dyn_cast(a.getAtom().getType()); + Type in = a.getInput().getType(); + if (!tiled || + !llvm::isa(tiled.getCopyAtom()) || + !llvm::isa(in)) + return emitOptionalError( + loc, "copy fragment inference covers tiled SIMT copies only"); + out.push_back(in); + return success(); +} + +// The atom's shape per CTA, then how often the atom repeats over the tile. +LogicalResult TiledMmaPartitionShapeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto shape = llvm::dyn_cast(a.getInput().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + return infer(loc, out, [&]() -> Type { + if (!atom || !shape || !which || !cg::holds_vector(shape.getRef()) || + cg::rank(shape.getRef()) != 2) + return {}; + int64_t full[2], part[2]; + switch (which.getInt()) { + case 0: + full[0] = atom->m; + full[1] = atom->k; + part[0] = atom->m / atom->ctas; + break; + case 1: + full[0] = atom->n; + full[1] = atom->k; + part[0] = atom->n / atom->ctas; + break; + case 2: + full[0] = atom->m; + full[1] = atom->n; + part[0] = atom->m / atom->ctas; + break; + default: + return {}; + } + part[1] = full[1]; + std::string rest; + for (int i = 0; i < 2; ++i) { + auto n = cg::size(shape.getRef(), i); + if (!cg::holds_int(n) || n.as_int64() % full[i]) + return {}; + rest += "," + std::to_string(n.as_int64() / full[i]); + } + auto r = + cg::from_string("((" + std::to_string(part[0]) + "," + + std::to_string(part[1]) + ")" + rest + ")"); + return r ? ShapeType::get(ctx, std::move(*r)) : Type(); + }); +} + +// ---------------------------------------------------------------- copy atoms + +namespace { + +std::optional layoutOf(const std::string &text) { + return cg::from_string(text); +} + +template int64_t asInt(const V &v) { + return cg::holds_int(v) ? v.as_int64() : -1; +} + +// TMEM addresses count lanes in the upper half-word: 2^21 bits per lane. +constexpr int64_t kTmemLaneBits = int64_t(1) << 21; + +// A copy atom's traits as CuTe spells them (cute/atom/copy_traits_*.hpp): +// the thread count and the (thr,val) -> bit layouts, and for tensor memory +// atoms the bit -> address layout. +struct CopyTraits { + int64_t threads = 1; + int64_t bits = 8; + cg::layout src, dst, ref, valId; +}; + +std::optional copyTraitsOf(Type atom) { + CopyTraits t; + auto set = [&](std::string src, std::string dst, bool refIsSrc, + std::string valId = "") -> std::optional { + auto s = layoutOf(src), d = layoutOf(dst); + if (!s || !d) + return std::nullopt; + t.src = *s; + t.dst = *d; + t.ref = refIsSrc ? *s : *d; + if (!valId.empty()) { + auto v = layoutOf(valId); + if (!v) + return std::nullopt; + t.valId = *v; + } + return t; + }; + auto n = [](int64_t v) { return std::to_string(v); }; + if (auto u = llvm::dyn_cast(atom)) { + t.bits = bitsOfType(u.getValType()); + std::string l = + "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; + return set(l, l, true); + } + if (auto s = llvm::dyn_cast(atom)) { + // SM90_U32xN_STSM_N and SM90_U16x2N_STSM_T: the SM75 ldmatrix layouts + // with source and destination swapped. + t.bits = bitsOfType(s.getValType()); + t.threads = 32; + int64_t matrices = s.getNumMatrices(); + bool transposed = bool(s.getTranspose()); + std::string ldsmSrc, ldsmDst; + if (!transposed) { + switch (matrices) { + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "(32,32):(32,1)"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "(32,(32,2)):(32,(1,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "(32,(32,4)):(32,(1,1024))"; + break; + default: + return std::nullopt; + } + } else { + switch (matrices) { + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; + break; + default: + return std::nullopt; + } + } + return set(ldsmDst, ldsmSrc, true); + } + // Tensor memory loads and stores: the tmem side is the reference; a store + // is a load with source and destination swapped. + auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, + bool load) -> std::optional { + t.bits = bitsOfType(val); + t.threads = 32; + std::string mem, reg, valId; + if (dp == 32 && bit == 32) { + mem = "(32," + n(1024 * x) + "):(0,1)"; + reg = "(32," + n(32 * x) + "):(" + n(32 * x) + ",1)"; + valId = "(" + n(32 * x) + ",32):(1," + n(kTmemLaneBits) + ")"; + } else if (dp == 16 && bit == 256) { + mem = "(32," + n(4096 * x) + "):(0,1)"; + reg = x == 1 ? "((4,8),(64,2)):((64,256),(1,2048))" + : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + + "),(1," + n(2048 * x) + ",256))"; + valId = "(" + n(256 * x) + ",16):(1," + n(kTmemLaneBits) + ")"; + } else { + return std::nullopt; + } + return load ? set(mem, reg, true, valId) : set(reg, mem, false, valId); + }; + // 16-bit packing and split reductions are not covered. + if (auto l = llvm::dyn_cast(atom)) + if (!l.getPack16b()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); + if (auto l = llvm::dyn_cast(atom)) + if (!l.getHalfSplitOff()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); + if (auto st = llvm::dyn_cast(atom)) + if (!st.getExpand16b()) + return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), + st.getNumRep(), false); + if (auto s = llvm::dyn_cast(atom)) { + // : SM100_UTCCP_4x32dp128bit_{C}cta. + t.bits = bitsOfType(s.getValType()); + int64_t dp = s.getNumDp(), bit = s.getNumBit(); + t.threads = s.getNumCta(); + if (dp != 32 || bit != 128 || + s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || t.threads < 1) + return std::nullopt; + return set("(" + n(t.threads) + ",(32,128,4)):(0,(1,32,0))", + "(" + n(t.threads) + ",16384):(0,1)", false, + "(32,128,4):(" + n(kTmemLaneBits) + ",1," + + n(32 * kTmemLaneBits) + ")"); + } + return std::nullopt; +} + +// A tiled copy: its atom in element units, the (thr,val) -> tile layout and +// the tiler over MN. +struct TiledCopy { + Type atom; + CopyTraits traits; + cg::layout tv; + cg::tile tiler; + int64_t atomThreads = 0, atomVals = 0, tiledThreads = 0; +}; + +std::optional tiledCopyOf(Type t) { + auto tiled = llvm::dyn_cast(t); + if (!tiled) + return std::nullopt; + auto traits = copyTraitsOf(tiled.getCopyAtom()); + if (!traits) + return std::nullopt; + TiledCopy tc; + tc.atom = tiled.getCopyAtom(); + tc.traits = *traits; + // Copy_Atom recasts the bit layouts to the value type. + tc.traits.src = cg::upcast(traits->bits, traits->src); + tc.traits.dst = cg::upcast(traits->bits, traits->dst); + tc.traits.ref = cg::upcast(traits->bits, traits->ref); + if (!cg::is_valid(tc.traits.src) || !cg::is_valid(tc.traits.dst) || + !cg::is_valid(tc.traits.ref)) + return std::nullopt; + tc.tv = tiled.getLayoutCopyTv().getRef(); + tc.tiler = tiled.getTilerMn().getRef(); + tc.atomThreads = asInt(cg::size(tc.traits.ref, 0)); + tc.atomVals = asInt(cg::size(tc.traits.ref, 1)); + tc.tiledThreads = asInt(cg::size(tc.tv, 0)); + if (tc.atomThreads <= 0 || tc.atomVals <= 0 || tc.tiledThreads <= 0) + return std::nullopt; + return tc; +} + +// The modes of a layout, one leaf mode as itself. +std::vector modesOf(const cg::layout &l) { + std::vector out; + if (!cg::holds_vector(l.shape())) + return {l}; + for (size_t i = 0; i < cg::rank(l.shape()); ++i) + out.push_back(cg::get(l, i)); + return out; +} + +// `l` with its first mode composed with `f` and the others kept: CuTe's +// tensor.compose(f, _). +cg::layout composeFirst(const cg::layout &l, const cg::layout &f) { + std::vector modes = modesOf(l); + modes[0] = cg::composition(modes[0], f); + return cg::make_layout(modes); +} + +// ((a,b),(c,d)) -> ((a,c),(b,d)) +cg::layout zip2(const cg::layout &l) { + return cg::make_layout(std::vector{ + cg::make_layout( + std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), + cg::make_layout( + std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); +} + +cg::coord profileOf(const char *text) { + return *cg::from_string(text); +} + +// TiledCopy::tile2thrfrg: ((Tile),(Rest...)) -> (Thr, (FrgV, FrgX), Rest...) +// through the atom's (src or dst) thread-value map `ref2trg`. +cg::layout tile2thrfrg(const TiledCopy &tc, const cg::layout &tiled, + const cg::layout &ref2trg, bool flatRest = true) { + auto atomShape = + cg::from_string("(" + std::to_string(tc.atomThreads) + "," + + std::to_string(tc.atomVals) + ")"); + if (!atomShape) + return cg::layout(cg::cg_error_t{}); + cg::layout atomTV = cg::zipped_divide(tc.tv, *atomShape); + if (!cg::is_valid(atomTV)) + return atomTV; + cg::layout trgTV = composeFirst(atomTV, ref2trg); + cg::layout thrval2mn = cg::coalesce(zip2(trgTV), profileOf("(1,(1,1))")); + if (!cg::is_valid(thrval2mn)) + return thrval2mn; + cg::layout tv = cg::composition(cg::get(tiled, 0), thrval2mn); + if (!cg::is_valid(tv)) + return tv; + std::vector modes{cg::get(tv, 0), cg::get(tv, 1)}; + // The rest modes follow; a rank-1 tensor keeps its one rest mode whole. + if (!flatRest) + modes.push_back(cg::get(tiled, 1)); + else + for (const cg::layout &m : modesOf(cg::get(tiled, 1))) + modes.push_back(m); + return cg::make_layout(modes); +} + +// TiledCopy::tidfrg_S/D: (M,N,...) -> (Thr, (FrgV, FrgX), (RestM, RestN, ...)). +cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { + cg::layout tiled = cg::zipped_divide(tensor, tc.tiler); + if (!cg::is_valid(tiled)) + return tiled; + cg::layout ref2trg = cg::composition(cg::right_inverse(tc.traits.ref), + src ? tc.traits.src : tc.traits.dst); + bool flatRest = + cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; + return tile2thrfrg(tc, tiled, ref2trg, flatRest); +} + +// ThrCopy::partition_S/D: the thread's slice of tidfrg. +Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, + const cg::coord &thread, bool src) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) + return {}; + cg::layout tv = tidfrg(tc, layout.getRef(), src); + if (!cg::is_valid(tv)) + return {}; + std::string coord = "(" + cg::to_string(thread); + for (size_t i = 1; i < cg::rank(tv.shape()); ++i) + coord += ",_"; + coord += ")"; + auto crd = cg::from_string(coord); + if (!crd) + return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, tv); + if (!cg::is_valid(sliced)) + return {}; + // The tensor's modes beyond the tiler are untouched and keep their strides. + size_t in = cg::rank(layout.getRef().shape()), r = cg::rank(tc.tiler); + size_t keep = + cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; + Type result = LayoutType::get(ctx, zeroUnitModesHead(sliced, keep, false)); + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return rebuild(ctx, c, result); +} + +// The shapes of a tiler's modes, as one shape. +std::optional tilerShape(const cg::tile &tiler) { + std::string text = cg::to_string(tiler); + if (text.size() < 2 || text.front() != '[') + return std::nullopt; + SmallVector parts; + StringRef(text).drop_front().drop_back().split(parts, ';'); + std::string shape = "("; + for (size_t i = 0; i < parts.size(); ++i) { + auto l = cg::from_string(parts[i].str()); + if (!l) + return std::nullopt; + shape += (i ? "," : "") + cg::to_string(l->shape()); + } + return cg::from_string(shape + ")"); +} + +// TiledCopy::retile: (V, RestM, RestN, ...) of one thread's values regrouped +// into the atom's values. +Type retileCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) + return {}; + const cg::layout &l = layout.getRef(); + size_t R = cg::rank(l.shape()); + int64_t V = asInt(cg::size(l, 0)); + auto shape = tilerShape(tc.tiler); + if (V <= 0 || !shape) + return {}; + cg::layout inv = cg::right_inverse(tc.tv).with_shape(*shape); + cg::layout frgMN = cg::upcast(tc.tiledThreads * V, inv); + if (!cg::is_valid(frgMN)) + return {}; + auto vLayout = layoutOf(std::to_string(V) + ":1"); + auto atomVals = layoutOf(std::to_string(tc.atomVals) + ":1"); + if (!vLayout || !atomVals) + return {}; + cg::layout frgV = cg::zipped_divide( + cg::logical_product(*vLayout, cg::right_inverse(frgMN)), *atomVals); + if (!cg::is_valid(frgV)) + return {}; + std::string tiler = "(" + std::to_string(V); + auto each = cg::product_each(frgMN.shape()); + for (size_t i = 0; i < cg::rank(each); ++i) + tiler += "," + cg::to_string(each[i]); + auto tilerShape = cg::from_string(tiler + ")"); + if (!tilerShape) + return {}; + cg::layout tiled = cg::zipped_divide(l, *tilerShape); + if (!cg::is_valid(tiled)) + return {}; + cg::layout v = composeFirst(tiled, frgV); + if (!cg::is_valid(v)) + return {}; + // v(_, (0, _, _...)): the first rest mode is the one the tile added. + std::vector modes{cg::get(v, 0)}; + std::vector rest = modesOf(cg::get(v, 1)); + for (size_t i = 1; i < rest.size() && i < R; ++i) + modes.push_back(rest[i]); + return rebuild(ctx, c, + LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); +} + +// make_cotiled_copy: the (thr,val) -> tile layout and the tiler of a copy +// whose atom addresses `data` directly. +std::optional> +cotiledCopy(const cg::layout &atomTV, const cg::layout &data) { + auto unit = layoutOf("1:0"); + cg::layout invData = + cg::make_layout(std::vector{cg::left_inverse(data), *unit}); + cg::layout tvData = cg::composition(invData, atomTV); + if (!cg::is_valid(tvData)) + return std::nullopt; + auto flat = cg::product_each(data.shape()); + size_t R = cg::rank(flat); + std::string flatText = cg::to_string(flat); + std::string tilerText = "["; + for (size_t i = 0; i < R; ++i) { + std::string stride = "("; + for (size_t j = 0; j < R; ++j) + stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); + auto pick = layoutOf(flatText + ":" + stride + ")"); + if (!pick) + return std::nullopt; + cg::layout mode = cg::filter(cg::composition(*pick, tvData)); + if (!cg::is_valid(mode)) + return std::nullopt; + tilerText += (i ? ";" : "") + cg::to_string(mode); + } + auto tiler = cg::from_string(tilerText + "]"); + auto flatShape = cg::from_string(flatText); + if (!tiler || !flatShape) + return std::nullopt; + cg::layout tile2data = cg::composition(cg::layout(*flatShape), *tiler); + cg::layout tv = cg::composition(cg::left_inverse(tile2data), tvData); + if (!cg::is_valid(tv)) + return std::nullopt; + return std::make_pair(tv, *tiler); +} + +Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, + const cg::tile &tiler) { + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, tv), + TileAttr::get(ctx, tiler)); +} + +// ---------------------------------------------------------------- TMA + +struct TmaFormat { + std::string name; + int64_t bits = 0; +}; + +// The descriptor's data format: the attribute when given, else the element's. +std::optional tmaFormatOf(Type elem, Attribute attr) { + TmaFormat f; + if (attr) { + auto format = llvm::dyn_cast(attr); + if (!format) + return std::nullopt; + f.name = cute_nvgpu::stringifyTmaDataFormat(format.getValue()).str(); + StringRef n = f.name; + if (n == "F16_RN" || n == "BF16_RN") + f.bits = 16; + else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") + f.bits = 32; + else if (n == "F64_RN" || n == "INT64") + f.bits = 64; + else if (!n.drop_until(llvm::isDigit) + .take_while(llvm::isDigit) + .getAsInteger(10, f.bits)) { + } else + return std::nullopt; + return f; + } + f.bits = bitsOfType(elem); + if (elem.isTF32()) + f.name = "TF32_RN"; + else if (elem.isF16()) + f.name = "F16_RN"; + else if (elem.isBF16()) + f.name = "BF16_RN"; + else if (elem.isF32()) + f.name = "F32_RN"; + else if (elem.isF64()) + f.name = "F64_RN"; + else + f.name = "U" + std::to_string(f.bits); + return f; +} + +// CuTe's coalesce_256: which adjacent modes of a flattened layout merge into +// one TMA box dimension, the merged extent staying within 256. +std::vector> coalesce256Groups(const cg::layout &l) { + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (!cg::holds_vector(fs)) + return {{0}}; + std::vector> groups; + std::vector cur{0}; + int64_t curS = asInt(fs[0]), curD = asInt(ft[0]); + for (size_t i = 1; i < cg::rank(fs); ++i) { + int64_t s = asInt(fs[i]), d = asInt(ft[i]); + if (curS == 1) { + cur = {i}; + curS = s; + curD = d; + } else if (s == 1) { + continue; + } else if (curS > 0 && curD >= 0 && s > 0 && d >= 0 && curS * curD == d && + s * curS <= 256) { + cur.push_back(i); + curS *= s; + } else { + groups.push_back(cur); + cur = {i}; + curS = s; + curD = d; + } + } + groups.push_back(cur); + return groups; +} + +// The layout with `shape`'s leaves regrouped by `groups`, over `stride`'s +// leaves: what composing with the box shape does, keeping every basis. +std::optional +regroup(const cg::layout &shape, const cg::layout &stride, + const std::vector> &groups) { + auto fs = cg::flatten(shape.shape()); + auto ft = cg::flatten(stride.stride()); + auto leaf = [&](const auto &flat, size_t i) { + return cg::holds_vector(flat) ? cg::to_string(flat[i]) + : cg::to_string(flat); + }; + std::string shapes, strides; + for (size_t g = 0; g < groups.size(); ++g) { + if (g) { + shapes += ","; + strides += ","; + } + if (groups[g].size() == 1) { + shapes += leaf(fs, groups[g][0]); + strides += leaf(ft, groups[g][0]); + continue; + } + shapes += "("; + strides += "("; + for (size_t k = 0; k < groups[g].size(); ++k) { + if (k) { + shapes += ","; + strides += ","; + } + shapes += leaf(fs, groups[g][k]); + strides += leaf(ft, groups[g][k]); + } + shapes += ")"; + strides += ")"; + } + return layoutOf("(" + shapes + "):(" + strides + ")"); +} + +// A layout composed with a coordinate layout: each mode of `rhs` selects, +// through its basis stride, the mode of `lhs` it indexes. +cg::layout composeCoord(const cg::layout &lhs, const cg::layout &rhs) { + std::vector modes; + for (const cg::layout &m : modesOf(rhs)) { + const auto &d = m.stride(); + if (cg::holds_int(d)) { + modes.push_back(m); + continue; + } + if (!cg::holds_scaled_basis(d)) + return cg::layout(cg::cg_error_t{}); + const auto &basis = std::get(d); + if (!basis.value_holds_int()) + return cg::layout(cg::cg_error_t{}); + auto scale = layoutOf(cg::to_string(m.shape()) + ":" + + std::to_string(basis.static_integral_value())); + if (!scale) + return cg::layout(cg::cg_error_t{}); + cg::layout sub = cg::get(lhs, basis.modes()); + cg::layout composed = cg::composition(sub, *scale); + if (!cg::is_valid(composed)) + return composed; + modes.push_back(composed); + } + return cg::make_layout(modes); +} + +// Leaves of congruent shape, stride and basis, with the path to each. +template +void walkLeaves(const cg::shape &s, const cg::stride &d, const cg::stride &e, + std::vector &path, F &&f) { + if (cg::holds_vector(s)) { + for (size_t i = 0; i < cg::rank(s); ++i) { + path.push_back(i); + walkLeaves(s[i], d[i], e[i], path, f); + path.pop_back(); + } + return; + } + f(s, d, e, path); +} + +bool basisAt(const cg::stride &leaf, const std::vector &path) { + return cg::holds_scaled_basis(leaf) && + std::get(leaf).modes() == path; +} + +// Which mode of a stride tuple carries the basis `path`, if any. +std::optional modeWithBasis(const cg::stride &strides, + const std::vector &path) { + for (size_t j = 0; j < cg::rank(strides); ++j) { + auto flat = cg::flatten(strides[j]); + if (!cg::holds_vector(flat)) { + if (basisAt(flat, path)) + return j; + continue; + } + for (size_t k = 0; k < cg::rank(flat); ++k) + if (basisAt(flat[k], path)) + return j; + } + return std::nullopt; +} + +// construct_tma_gbasis: the TMA box shape with, per mode, the gmem modes it +// runs over. +cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, + const cg::layout &smem, const cg::layout &ctaVMap) { + auto fail = [] { return cg::layout(cg::cg_error_t{}); }; + cg::layout inv = cg::right_inverse(smem); + cg::layout full = cg::coalesce(cg::composition(ctaVMap, inv)); + if (!cg::is_valid(full)) + return full; + std::vector fullModes = modesOf(full); + // Keep the leading modes that step through gmem one element at a time. + size_t smemRank = 0; + for (; smemRank < fullModes.size(); ++smemRank) { + const auto &d = fullModes[smemRank].stride(); + if (!cg::holds_scaled_basis(d)) + break; + const auto &b = std::get(d); + if (!b.value_holds_int() || b.static_integral_value() != 1) + break; + } + if (smemRank == 0) + return fail(); + fullModes.resize(smemRank); + cg::layout sidx2gmode = cg::make_layout(fullModes); + cg::layout tileGstride = + cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); + if (!cg::is_valid(tileGstride)) + return tileGstride; + cg::layout gbasis = cg::make_identity_layout(gmem.shape()); + cg::layout tileGbasisTmp = composeCoord(gbasis, sidx2gmode); + if (!cg::is_valid(tileGbasisTmp)) + return tileGbasisTmp; + auto regrouped = + regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); + if (!regrouped) + return fail(); + cg::layout tmaGbasisTile = *regrouped; + // Gmem modes the tile misses still shape the descriptor, as size-1 modes. + cg::layout gmemT = cg::recast(tmaBits, elemBits, gmem); + if (!cg::is_valid(gmemT)) + return gmemT; + std::vector modes = modesOf(tmaGbasisTile); + auto flatTile = cg::flatten(tmaGbasisTile.stride()); + std::vector path; + bool ok = true; + walkLeaves(gmemT.shape(), gmemT.stride(), gbasis.stride(), path, + [&](const auto &s, const auto &d, const auto &e, + const std::vector &p) { + if (asInt(s) == 1 || asInt(d) == 0) + return; + bool found = false; + if (!cg::holds_vector(flatTile)) { + found = basisAt(flatTile, p); + } else { + for (size_t k = 0; k < cg::rank(flatTile) && !found; ++k) + found = basisAt(flatTile[k], p); + } + if (found) + return; + auto extra = layoutOf("1:" + cg::to_string(e)); + if (!extra) + ok = false; + else + modes.push_back(*extra); + }); + if (!ok) + return fail(); + cg::layout result = cg::make_layout(modes); + if (modes.size() > 4) + result = cg::group(4, modes.size(), result); + return result; +} + +// The TMA tensor's layout: gmem shape over TMA coordinate bases, scaled to +// the descriptor's element. +std::optional tmaTensorLayout(const cg::layout &gmem, + int64_t elemBits, int64_t tmaBits, + const cg::layout &gbasis) { + cg::layout basis = cg::make_identity_layout(gmem.shape()); + const auto &tmaStrides = gbasis.stride(); + bool ok = true; + std::string text; + std::vector path; + // Written in the shape's own nesting. + std::function + write = [&](const cg::shape &s, const cg::stride &d, + const cg::stride &e) { + if (cg::holds_vector(s)) { + text += "("; + for (size_t i = 0; i < cg::rank(s); ++i) { + if (i) + text += ","; + path.push_back(i); + write(s[i], d[i], e[i]); + path.pop_back(); + } + text += ")"; + return; + } + if (asInt(s) == 1 || asInt(d) == 0) { + text += "0"; + return; + } + auto j = cg::holds_vector(tmaStrides) + ? modeWithBasis(tmaStrides, path) + : (basisAt(tmaStrides, path) ? std::optional(0) + : std::nullopt); + if (!j) { + text += "0"; + return; + } + if (*j == 0) { + int64_t stride = asInt(d); + if (stride <= 0) { + ok = false; + return; + } + int64_t num = stride * elemBits, den = tmaBits; + int64_t g = std::gcd(num, den); + num /= g; + den /= g; + text += std::to_string(num) + + (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; + return; + } + auto mode = cg::holds_vector(tmaStrides) ? tmaStrides[*j] : tmaStrides; + auto flat = cg::flatten(mode); + if (cg::holds_vector(flat) && cg::rank(flat) != 1) { + ok = false; + return; + } + text += "1@" + std::to_string(*j); + }; + write(gmem.shape(), gmem.stride(), basis.stride()); + if (!ok) + return std::nullopt; + return layoutOf(cg::to_string(gmem.shape()) + ":" + text); +} + +// The smem tile layout without its swizzle. +std::optional plainLayoutOf(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); + return std::nullopt; +} + +struct TmaAtomInfo { + int64_t elemBits = 0, copyBits = 0; +}; + +std::optional tmaAtomInfoOf(Type atom) { + TmaAtomInfo info; + if (auto l = + llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else if (auto r = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(r.getValType()); + info.copyBits = r.getCopyBits(); + } else if (auto l = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else { + return std::nullopt; + } + if (info.elemBits <= 0 || info.copyBits <= 0) + return std::nullopt; + return info; +} + +// make_non_exec_tiled_tma_load/store: the atom and the TMA tensor. +LogicalResult inferTmaAtom(MLIRContext *ctx, std::optional loc, + Type gmemType, Type smemType, Type ctaVMapType, + Attribute kind, Attribute format, + SmallVectorImpl &out) { + auto gmem = llvm::dyn_cast(gmemType); + auto smem = plainLayoutOf(smemType); + auto ctaVMap = llvm::dyn_cast(ctaVMapType); + if (!gmem || !smem || !ctaVMap) + return emitOptionalError( + loc, "expects a gmem tensor, an smem layout and a CTA value map"); + auto glayout = llvm::dyn_cast(gmem.getLayout()); + if (!glayout) + return emitOptionalError(loc, "gmem tensor needs a plain layout"); + Type elem = gmem.getValueType(); + auto fmt = tmaFormatOf(elem, format); + if (!fmt || fmt->bits <= 0) + return emitOptionalError(loc, "unknown TMA data format"); + int64_t elemBits = bitsOfType(elem); + cg::layout gbasis = + tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); + if (!cg::is_valid(gbasis)) + return emitOptionalError(loc, "cannot build the TMA basis"); + auto tensor = tmaTensorLayout(glayout.getRef(), elemBits, fmt->bits, gbasis); + if (!tensor) + return emitOptionalError(loc, "cannot build the TMA tensor"); + int64_t copyBits = asInt(cg::size(gbasis)) * fmt->bits; + auto gbasisType = LayoutType::get(ctx, gbasis); + std::optional dataFormat = + cute_nvgpu::symbolizeTmaDataFormat(fmt->name); + Type atom; + if (auto k = llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); + } else if (auto k = + llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); + } else { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get( + ctx, elem, copyBits, gbasisType, dataFormat); + } + std::string zeros = "("; + for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) + zeros += i ? ",0" : "0"; + auto iter = cg::from_string(zeros + ")"); + if (!iter) + return emitOptionalError(loc, "cannot build the TMA coordinate"); + out.push_back(atom); + out.push_back(CoordTensorType::get(ctx, IntTupleType::get(ctx, *iter), + LayoutType::get(ctx, *tensor))); + return success(); +} + +} // namespace + +// ---------------------------------------------------------------- cute copy +// ops + +// TiledCopy::get_layoutD_TV over the reference tile, then the tiled copy of +// another atom with that layout (make_tiled_copy_D). +Type mlir::cutlass_compiler::cute::tiledCopyD(Type atom, TiledCopyType tiled) { + auto tc = tiledCopyOf(tiled); + if (!tc) + return {}; + auto shape = tilerShape(tc->tiler); + if (!shape) + return {}; + auto refShape = + cg::from_string("(" + cg::to_string(*shape) + ",1)"); + if (!refShape) + return {}; + cg::layout refD(*refShape); + cg::layout ref2trg = + cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); + cg::layout tv = tile2thrfrg(*tc, refD, ref2trg); + if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) + return {}; + cg::layout layoutTV = zeroUnitModes( + cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); + MLIRContext *ctx = atom.getContext(); + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, layoutTV), + tiled.getTilerMn()); +} + +LogicalResult TiledCopyPartitionSOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), true); + }); +} + +LogicalResult TiledCopyPartitionDOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), false); + }); +} + +LogicalResult TiledCopyRetileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + if (!tc || !c) + return emitOptionalError(loc, "expects a tiled copy and a tensor"); + return infer(loc, out, [&] { return retileCopy(ctx, *tc, *c); }); +} + +// ---------------------------------------------------------------- cute_nvgpu + +namespace mlir::cutlass_compiler::cute_nvgpu { + +LogicalResult AtomSetValueOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// The executable atom of a TMA descriptor: the element, copy size and basis +// carry over; the kind becomes the mode and, for loads, the CTA count. +LogicalResult AtomCopyMakeExecTmaOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool over = a.getOverride(), noOob = a.getNoFullyOobTile(); + Type in = a.getInput().getType(); + auto stride = cg::from_string("()"); + if (!stride) + return emitOptionalError(loc, "cannot build the TMA stride"); + auto gStride = StrideType::get(ctx, *stride); + if (auto load = llvm::dyn_cast(in)) { + TiledTmaLoad kind = load.getKind(); + bool twoSm = kind == TiledTmaLoad::sm_100_2sm || + kind == TiledTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == TiledTmaLoad::sm_90_multicast || + kind == TiledTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomTmaLoadType::get( + ctx, load.getValType(), /*sparsity=*/1, load.getCopyBits(), + TmaLoadMode::tiled, twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis(), + over, noOob)); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaStoreType::get( + ctx, store.getValType(), /*sparsity=*/1, store.getCopyBits(), + TmaStoreMode::tiled, gStride, store.getTmaGbasis(), over, noOob)); + return success(); + } + if (auto red = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaReduceType::get( + ctx, red.getValType(), red.getCopyBits(), TmaStoreMode::tiled, + red.getKind(), gStride, red.getTmaGbasis())); + return success(); + } + if (auto load = llvm::dyn_cast(in)) { + Im2ColTmaLoad kind = load.getKind(); + bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomIm2ColTmaLoadType::get( + ctx, load.getValType(), load.getCopyBits(), twoSm ? 2 : 1, gStride, + mcast, load.getTmaGbasis())); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), + store.getCopyBits(), gStride, + store.getTmaGbasis())); + return success(); + } + // The gather/scatter atoms' executable forms are not recovered. + return emitOptionalError(loc, + "expects a non-executable tiled or im2col TMA atom"); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaLoadOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaStoreOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + /*kind=*/nullptr, a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaReduceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +// tma_partition: the smem tile's contiguous vectors, one instruction each, +// read through every tensor; a multicast CTA starts at its share. +LogicalResult AtomTmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto info = tmaAtomInfoOf(a.getTmaAtom().getType()); + auto coord = llvm::dyn_cast(a.getCtaCoord().getType()); + auto ctaLayout = llvm::dyn_cast(a.getCtaLayout().getType()); + auto smem = carrierOf(a.getSmemTensor().getType()); + if (!info || !coord || !ctaLayout || !smem) + return emitOptionalError( + loc, "expects a TMA atom, a CTA coord and layout, and tensors"); + auto smemLayout = llvm::dyn_cast_or_null(smem->layout); + if (!smemLayout || !cg::holds_vector(smemLayout.getRef().shape())) + return emitOptionalError(loc, + "smem tensor needs a plain layout of rank >= 1"); + int64_t numVal = info->copyBits / info->elemBits; + cg::layout inv = cg::right_inverse(cg::get(smemLayout.getRef(), 0)); + int64_t b = asInt(cg::size(inv)); + int64_t n = asInt(cg::size(smemLayout.getRef(), 0)); + if (b <= 0 || n <= 0 || n % b || numVal <= 0) + return emitOptionalError(loc, "cannot tile the smem vector"); + // tile_to_shape: the vector repeated over the tile, broadcast modes at 0. + cg::layout layoutV = inv; + if (n != b) { + auto rest = layoutOf(std::to_string(n / b) + ":1"); + layoutV = cg::logical_product(inv, *rest); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot tile the smem vector"); + } + auto tmaV = layoutOf(std::to_string(numVal) + ":1"); + layoutV = cg::logical_divide(layoutV, *tmaV); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot divide the smem vector"); + // The multicast offset inside the instruction's vector. + int64_t ctas = asInt(cg::cosize(ctaLayout.getRef())); + std::optional multicast; + if (ctas > 1) { + // The CTA layout scaled by the share: (cosize:share) o cta_layout. + auto share = + layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); + cg::layout scaled = cg::composition(*share, ctaLayout.getRef()); + if (!cg::is_valid(scaled)) + return emitOptionalError(loc, "cannot scale the CTA layout"); + multicast = cg::layout_eval(coord.getRef(), scaled); + } + SmallVector results; + auto partition = [&](Type tensor) -> Type { + auto c = carrierOf(tensor); + if (!c) + return {}; + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), + profileOf("((1,1))")); + if (!cg::is_valid(v)) + return {}; + // Only the vector's unit modes get a zero stride; the rest keep theirs. + Type result = LayoutType::get( + ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); + if (!multicast) + return rebuild(ctx, *c, result); + // The index split over the vector's modes, column-major, keeping what + // its divisibility says of each coordinate. + auto vec = cg::get(v.shape(), 0); + if (!cg::holds_vector(vec)) + return {}; + auto split = splitIndex(*multicast, vec[0]); + if (!split) + return {}; + std::string crd = "((" + *split + ",0)"; + for (size_t i = 1; i < cg::rank(v.shape()); ++i) + crd += ",0"; + auto at = cg::from_string(crd + ")"); + if (!at) + return {}; + cg::int_tuple offset = cg::layout_eval(*at, v); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return rebuild(ctx, *c, result); + }; + results.push_back(partition(a.getSmemTensor().getType())); + for (Value target : a.getTargetTensors()) + results.push_back(partition(target.getType())); + for (Type t : results) + if (!t) + return emitOptionalError(loc, "cannot partition for TMA"); + out.append(results.begin(), results.end()); + return success(); +} + +namespace { +// make_cotiled_copy over a tensor memory tensor for an atom whose thread +// layout is `atomT`. +Type tmemTiledCopy(MLIRContext *ctx, std::optional loc, Type atom, + Type tmem, const std::string &atomT, + SmallVectorImpl &out) { + auto traits = copyTraitsOf(atom); + auto m = llvm::dyn_cast(tmem); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!traits || !layout || !cg::is_valid(traits->valId)) + return {}; + auto t = layoutOf(atomT); + if (!t) + return {}; + cg::layout v = cg::coalesce(cg::upcast(traits->bits, traits->valId)); + if (!cg::is_valid(v)) + return {}; + auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), + layout.getRef()); + if (!tiled) + return {}; + return tiledCopyType(ctx, atom, tiled->first, tiled->second); +} +} // namespace + +// make_tmem_copy: four warps, each at the same place in its own lane quarter. +LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getAtom().getType()); + if (!traits || traits->bits <= 0) + return emitOptionalError(loc, "expects a tmem atom"); + int64_t lane = kTmemLaneBits / traits->bits; + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getAtom().getType(), + a.getTmemMemref().getType(), + "(32,4):(0," + std::to_string(32 * lane) + ")", out); + }); +} + +// make_s2t_copy (make_utccp_copy): one thread per CTA. +LogicalResult AtomCopyMakeS2TCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getCopyS2tAtom().getType()); + if (!traits) + return emitOptionalError(loc, "expects an s2t atom"); + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getCopyS2tAtom().getType(), + a.getTmemMemref().getType(), + std::to_string(traits->threads) + ":0", out); + }); +} + +namespace { +// The smem side of an s2t copy as descriptor units: the tile recast to 128 +// bits, viewed through `desc`. +LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, + Type desc, SmallVectorImpl &out) { + if (auto tiled = llvm::dyn_cast(atom)) + atom = tiled.getCopyAtom(); + auto m = llvm::dyn_cast(view); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!llvm::isa(atom) || !layout) + return emitOptionalError(loc, "expects an s2t atom and an smem tensor"); + // The compiler wants mode 0 as (values, 1). + const cg::layout &l = layout.getRef(); + auto m0 = cg::get(l.shape(), 0); + if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || + !cg::holds_int(cg::size(m0, 1)) || cg::size(m0, 1).as_int64() != 1) + return emitOptionalError(loc, "expects mode 0 as (values, 1)"); + return infer(loc, out, [&]() -> Type { + cg::layout v = cg::recast(128, bitsOfType(m.getValueType()), l); + if (!cg::is_valid(v)) + return {}; + return SmemDescViewType::get(atom.getContext(), desc, + LayoutAttr::get(atom.getContext(), v)); + }); +} + +// An operation whose inference rule is not recovered: the compiler either +// aborts on the inputs tried or needs inputs no probe produced. +LogicalResult unrecovered(std::optional loc, StringRef op) { + return emitOptionalError(loc, op, " result inference is not recovered"); +} +} // namespace + +LogicalResult AtomGetCopyS2TSmemDescViewOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescType::get(ctx), out); +} + +LogicalResult AtomGetCopyS2TSmemDescViewSM107Op::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescSM107Type::get(ctx), out); +} + +// A generic-space pointer to the parameter, at its type's natural alignment. +LogicalResult GetGridConstantPointerOp::inferReturnTypes( + MLIRContext *ctx, std::optional, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type t = operands[0].getType(); + bool desc = llvm::isa(t); + out.push_back(PtrType::get(ctx, t, AddressSpace::generic, + desc ? 64 : PtrType::getNaturalAlignment(t), {}, + {})); + return success(); +} + +// The residues of the TMA tensor's coordinates: one per box mode, starting +// at the extent of the gmem mode it runs over and counting down. +LogicalResult AtomMakeTmaResidueTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type atom = a.getTmaAtom().getType(), gbasis; + if (auto l = llvm::dyn_cast(atom)) + gbasis = l.getTmaGbasis(); + if (auto s = llvm::dyn_cast(atom)) + gbasis = s.getTmaGbasis(); + auto basis = llvm::dyn_cast_or_null(gbasis); + auto m = llvm::dyn_cast(a.getGmemTensor().getType()); + auto gmem = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + return infer(loc, out, [&]() -> Type { + if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) + return {}; + const cg::stride &modes = basis.getRef().stride(); + // Each box mode's one basis: the gmem leaf it runs over. + std::vector> paths; + for (size_t j = 0; j < cg::rank(modes); ++j) { + auto flat = cg::flatten(modes[j]); + const cg::stride &leaf = + cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; + if (!cg::holds_scaled_basis(leaf)) + return {}; + paths.push_back(std::get(leaf).modes()); + } + std::vector iter(paths.size()), strides; + std::vector path; + const cg::layout &g = gmem.getRef(); + walkLeaves(g.shape(), g.stride(), g.stride(), path, + [&](const auto &s, const auto &, const auto &, + const std::vector &p) { + auto j = + std::find(paths.begin(), paths.end(), p) - paths.begin(); + if (j == static_cast(paths.size())) + return strides.push_back("0"); + iter[j] = cg::to_string(s); + strides.push_back("-1@" + std::to_string(j)); + }); + for (const std::string &e : iter) + if (e.empty()) + return {}; + size_t i = 0; + auto layout = cg::from_string(cg::to_string(g.shape()) + ":" + + nestLike(g.shape(), strides, i)); + auto start = + cg::from_string("(" + llvm::join(iter, ",") + ")"); + if (!layout || !start) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), + LayoutType::get(ctx, *layout)); + }); +} + +LogicalResult AtomGetCoordTensorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return unrecovered(loc, "cute_nvgpu.atom.get_coord_tensor"); +} + +// The im2col and gather/scatter TMA atoms and the circular descriptors +// validate inputs (canonical UMMA layouts, im2col corners) whose rules are +// not recovered. +#define CUTE_UNRECOVERED(Op, name) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, \ + PropertyRef, RegionRange, SmallVectorImpl &) { \ + return unrecovered(loc, name); \ + } +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_store") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, + "cute_nvgpu.sm103.make_umma_smem_desc_circular") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, + "cute_nvgpu.sm107.make_umma_smem_desc_circular") +#undef CUTE_UNRECOVERED + +} // namespace mlir::cutlass_compiler::cute_nvgpu diff --git a/mlir/dialects/cute_ir/CuteInference.h b/mlir/dialects/cute_ir/CuteInference.h new file mode 100644 index 00000000..c7837b20 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteInference.h @@ -0,0 +1,24 @@ +//===- CuteInference.h - result types the compiler infers -------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H +#define MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +namespace mlir::cutlass_compiler::cute { +// make_tiled_copy_D: `atom` tiled over `tiled`'s destination thread-value +// layout and tiler, or null when the atom is not one this tree knows. +Type tiledCopyD(Type atom, TiledCopyType tiled); + +// Result types of operations the compiler does not infer (their builders +// take the type): what the DSL passes, or null when not computable here. +Type makeLayoutResultType(Type shape, Type stride); // stride may be null +Type makeViewResultType(Type iter, Type layout); +Type getResultType(Type input, ArrayRef mode); +Type assumeResultType(Type src); +Type makeArithTupleIterResultType(Type tuple); +Type fastDivmodCreateDivisorResultType(Type divisor); +Type fastDivmodComputeResultType(Type divisor); // both results +} // namespace mlir::cutlass_compiler::cute + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H diff --git a/mlir/dialects/cute_ir/CuteOps.cpp b/mlir/dialects/cute_ir/CuteOps.cpp new file mode 100644 index 00000000..0c8ddaf3 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteOps.cpp @@ -0,0 +1,9 @@ +//===- CuteOps.cpp - the compiler's CuTe operations -----------------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +// Operand constraints name cute_nvgpu and LLVM types. +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_ir/CuteOps.cpp.inc" diff --git a/mlir/dialects/cute_ir/CuteOps.td b/mlir/dialects/cute_ir/CuteOps.td new file mode 100644 index 00000000..d2e342df --- /dev/null +++ b/mlir/dialects/cute_ir/CuteOps.td @@ -0,0 +1,3795 @@ +// The cute dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute.deref_desc_iter operand #0: no probe type accepted; any type allowed +// cute.make_desc_iter operand #0: no probe type accepted; any type allowed + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cute_32BitSignlessIntegerOr64BitSignless : Type, "32-bit signless integer or 64-bit signless integer">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or CuTe Layout Type or Cute shape type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or Cute coord_tensor type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor3 : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def Cute_ATupleIsAFixedLengthOrderedList : Type($_self))">, "A tuple is a fixed-length, ordered list consisting of multiple elements.">; +def Cute_ATypeInterfaceForDescriptorIteratorTypes : Type, "A type interface for descriptor iterator types.">; // not recovered: no pool type passed +def Cute_ATypeInterfaceForMemoryDescriptorsSuchAs : Type, "A type interface for memory descriptors such as smem desc etc.">; // not recovered: no pool type passed +def Cute_ArithmeticTupleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Arithmetic tuple type">; +def Cute_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance or CopyAtomTypeInterfaceV2 instance">; +def Cute_CopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self))">, "CopyAtomTypeInterfaceV2 instance">; +def Cute_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def Cute_CuTeIteratorTypeWithIntTupleAsIts : Type($_self))">, "CuTe iterator type with int_tuple as its underlying implementation">; +def Cute_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def Cute_CuTeLayoutTypeOrCuteSwizzleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or Cute swizzle type">; +def Cute_CuTeLayoutTypeOrSwizzleWithValidParameters : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or swizzle with valid parameters that can be represented by 32-bit masks">; +def Cute_CuTePointerTo32BitSignlessInteger : Type($_self))">, "CuTe pointer to 32-bit signless integer">; +def Cute_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def Cute_CuTePointerTypeOrAMemoryReferenceType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::FastDivmodDivisorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\")) || ::llvm::isa<::mlir::VectorType>($_self))">, " CuTe pointer type or A memory reference type that represents a tensor view with an iterator and layout or Integer type with constraints. or CuTe fast divmod divisor type or loadable maybe-static CuTe type with runtime storage or LLVM type with size">; +def Cute_CuTePointerTypeOrLLVMPointerType : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self))">, " CuTe pointer type or LLVM pointer type">; +def Cute_CuTeSparseElementType : Type($_self))">, " CuTe sparse element type">; +def Cute_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def Cute_CuteCoordType : Type($_self))">, "Cute coord type">; +def Cute_CuteShapeType : Type($_self))">, "Cute shape type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type or CuTe Layout Type or CuTe composed layout type or ViewTypeInterface instance">; +def Cute_CuteStrideType : Type($_self))">, "Cute stride type">; +def Cute_CuteTileType : Type($_self))">, "Cute tile type">; +def Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant : Type($_self))">, "Divisor for fast unsigned integer divmods by invariant integers.">; +def Cute_GeneralizedIteratorTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, " Generalized iterator type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, " Generalized layout type of CuTe or ViewTypeInterface instance">; +def Cute_I32I64OrConstrainedInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64))">, "i32, i64 or Constrained integer">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "IntTuple as integer or tuple of IntTuple(s) or Cute shape type">; +def Cute_IntTupleTypeThatDepthIs0 : Type($_self))">, "Int tuple type that depth is 0">; +def Cute_Integer : Type($_self))">, "integer">; +def Cute_IntegerOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "integer or Integer type with constraints. ">; +def Cute_IntegerTypeWithConstraints : Type($_self))">, " Integer type with constraints. ">; +def Cute_MaybeStaticTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MaybeStaticTypeInterface instance">; +def Cute_MemrefInGenericOrRegisterMemOrSmem : Type($_self))">, "memref in generic or register mem or smem">; +def Cute_MemrefInSmemWithStaticAndNonComposed : Type($_self))">, "memref in smem with static and non-composed layout">; +def Cute_MmaAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MmaAtomTypeInterface instance">; +def Cute_OfRanks1 : Type($_self))">, " of ranks 1">; +def Cute_OfRanks1OrFloatingPointOrInteger : Type($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::VectorType>($_self))">, " of ranks 1 or floating-point or integer">; +def Cute_PointerTypeInterfaceInstance : Type($_self))">, "PointerTypeInterface instance">; +def Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self))">, "PrefetchAtomTypeInterface or TiledCopyType with PrefetchAtomTypeInterface copy atom">; +def Cute_RefinableTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "RefinableTypeInterface instance">; +def Cute_RefinableTypeInterfaceInstanceOrInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "RefinableTypeInterface instance or integer">; +def Cute_StaticLayout : Type($_self))">, "static layout">; +def Cute_StaticShape : Type($_self))">, "static shape">; +def Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "Tile-like types, e.g. LayoutType, ShapeType, TileType">; +def Cute_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def Cute_TiledCopyAtomTypeOrTiledCopyV2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self))">, "Tiled copy atom type. or Tiled copy V2 atom type.">; +def Cute_TiledCopyV2AtomType : Type($_self))">, "Tiled copy V2 atom type.">; +def Cute_TiledMMAAtomType : Type($_self))">, "Tiled MMA atom type.">; +def Cute_TypeInterfaceForCuteTypesCanBePrinted : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self))">, "Type interface for cute types can be printed by `cute.print`">; +def Cute_ValueTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. ">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || $_self.isSignlessInteger(8) || $_self.isSignlessInteger(16) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64) || ::llvm::isa<::mlir::IndexType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. or 8/16/32/64-bit integer or index">; +def Cute_VectorOf1BitSignlessIntegerValuesOf : Type($_self))">, "vector of 1-bit signless integer values of ranks 1">; +def Cute_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe ">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe or Cute tile type">; +def Cute_XTupleLikeType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type">; +def Cute_XTupleLikeTypeOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type or CuTe Layout Type">; + +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class Cute_InferredOp traits = []> + : Cute_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cute_AddOffsetOp : Cute_InferredOp<"add_offset", [Pure]> { + let description = [{ + The `cute.add_offset` operation accepts an input IteratorTypeInterface and + a corresponding offset type which must be an + arithmetic tuple. It returns a new SSA value as type IteratorTypeInterface. + + When src operand is a sparse pointer, the logical offset must be a multiple + of the sparsity factor. The actual offset added to the src operand is + equal to the logical offset divided by the sparsity factor. + + `add_offset(%it, N)` returns a new iterator pointing to an object with an + offset of`N` elements. It is similar to C++'s iterator arithmetic. + + Example: + + ```mlir + // Add offset to 1-D linear pointer + %ptr = cute.add_offset(%0, %1) : (!cute.ptr>, !cute.int_tuple<"?">) -> !cute.ptr> + + // Add offset to 1-D linear pointer with static offset + %off = cute.static : !cute.int_tuple<"1012"> + %ptr = cute.add_offset(%0, %off) : (!cute.ptr>) -> !cute.ptr> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple + %arith_tuple = cute.add_offset(%0, %1) : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple with static offset + %arith_tuple = cute.add_offset(%0, "(3,(1,3))") : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$src, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$dst + ); +} +def Cute_AlgorithmCopyOp : Cute_Op<"copy", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's copy algorithm copies the elements of a source Tensor into the elements + of a destination Tensor. + + See the file cutlass/media/docs/cute/04_algorithms.md in the CUTLASS repository + for more information. + + A Tensor (cute.memref) encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, copy can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware copy instructions. + + An .copy accepts CuteCopyAtomTypeInterface as first parameter. To keep consistent + interface with CUTLASS C++, it could be either atom.tiled_copy or atom.universal_copy. + + Optional argument 'pred' is used to guard memory accesses performed in copy. + Predication works in granularity of atom_v. For src tensor whose shape is: + `((atom_v, rest_v), rest_m, rest_n, ...)`, the pred tensor shape should be: + `((rest_v), rest_m, rest_n, ...)`, we denoted this as the canonical format. + If the pred tensor's value equals 0, the corresponding copy will be skipped. + + When using atom.universal_copy instead of atom.tiled_copy, we allow the layout of + `pred` to deviate from the canonical format. Instead, the layout shape of `pred` + must match the layout shape of `src` or `dst`. Additionally, the `copy_bits` + parameter cannot be set for atom.universal_copy when predicate is present. + + The operands `src` and `dst` are variadic, each containing a variable number of tensors: + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Example: + + ```mlir + cute.copy(%tiled_copy, %src, %dst) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">, + !cute.memref, "(((64,128),1),1,1,2):(((1,64),0),0,0,8192)">) + + // atom.universal_copy with canonicalize layout format + cute.copy(%copy_atom, %src, %dst, %pred) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1),2,4):((0),1,2)">) + + // atom.universal_copy with relaxed layout format + cute.copy(%atom, %arg0, %arg1, %arg2) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "(4,4):(1,4)">, + !cute.memref, + !cute.memref, "(4,4):(1,4)">) + + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$copy_atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_AlgorithmGemmOp : Cute_Op<"gemm", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's gemm algorithm takes 1 mma atom and 4 inputs(cute.memref/cute_nvgpu.smem_desc_view) + as input. + It performs the general matrix multiplication: + `D += A * B + C`. + Layout carried by operads determines the exact form of the gemm computation. + Generally speaking, there are 5 valid forms: + 1. `(V) x (V) => (V)`. + The element-wise product of vectors. Dispatches to FMA or MMA. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 1`. + 2. `(M) x (N) => (M,N)`. + The outer product of vectors. Dispatches to [3] with new mode K=(1). + In this case, `rank(A) = rank(B) = 1` and `rank(C) = rank(D) = 2`. + 3. `(M,K) x (N,K) => (M,N)`. + The product of matrices. Dispatches to [5] with MMA vector-mode V. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 2`. + 4. `(V,M) x (V,N) => (V,M,N)`. + The batched outer product of vectors. Accounts for register reuse and dispatches to [1] for each (m,n). + In this case, `rank(A) = rank(B) = 2` and `rank(C) = rank(D) = 3`. + 5. `(V,M,K) x (V,N,K) => (V,M,N)`. + The batched product of matrices. Dispatches to [4] for each (k). + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 3`. + + Currently, only the form in item 5 is supported. + + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular GEMM, `a` and `b` contain the GEMM A and B tensors respectively. + - For GEMM with auxiliary operands, `a` and `b` contain the GEMM A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledGemm, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseGemm, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseGemm, `a` = [A, SFA, E] and `b` = [B, SFB]. + See the file cutlass/media/docs/cute/04_algorithms.md and include/cute/algorithm/gemm.hpp in the CUTLASS repository for more information. + + Example (regular GEMM): + ```mlir + cute.gemm(%mma, %d, %a, %b, %c) { + } : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">>, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">) + ``` + + Example (block-scaled GEMM): + ```mlir + cute.gemm(%mma, %d, [%a, %sfa], [%b, %sfb], %c) : ( + !cute.tiled_mma, + atom_layout_MNK = <"(1,2,1):(0,1,0)">>, + !cute.memref, "((128,128),1,1):((65536,1),0,0)">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + !cute.memref, "((128,128),1,1):((65536,1),0,0)">) + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$mma_atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_AlgorithmPrefetchOp : Cute_Op<"prefetch"> { + let summary = "CuTe's prefetch algorithm prefetches the elements of source Tensor(s) into cache."; + let description = [{ + A Tensor encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, prefetch can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware prefetch instructions. + + An .prefetch accepts CutePrefetchAtomTypeinterface as first parameter. + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`, + mirroring `cute.copy` for the same mode. + + Example: + + ```mlir + cute.prefetch(%tiled_copy, %src) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">) + + // gather4 mode: data tensor + gmem index tensor + cute.prefetch(%tiled_copy, %src_data, %src_idx) : ( + !cute.tiled_copy, ...>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$prefetch_atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_AppendToRankOp : Cute_InferredOp<"append_to_rank", [Pure]> { + let summary = "Append element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(5,4,3)"> + %elem = cute.make_shape() : !cute.shape<"(2,1)"> + %new_layout = cute.append_to_rank<5> (%shp, %elem) + : (..., ...) -> !cute.shape<"(5,4,3,(2,1),(2,1))"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ApplySwizzleOp : Cute_InferredOp<"apply_swizzle", [Pure]> { + let description = [{ + The `apply_swizzle` operation applies a swizzle transformation to a pointer's + address, producing a new pointer with the swizzled address. This is necessary + for dereferencing pointers that use swizzled memory layouts, such as those used + for improved memory access patterns on GPUs. + + Two forms are supported: + + 1. **CutePtrType form**: The swizzle is encoded in the pointer type itself. + No explicit swizzle operand is required or allowed. + + ```mlir + %swizzled = cute.apply_swizzle(%ptr) : !cute.ptr, S<3,4,3>> + ``` + + 2. **llvm.ptr form**: The swizzle is passed as an optional attribute in the + attribute dictionary. If the swizzle attribute is absent, the operation + is a no-op (identity swizzle). + + ```mlir + // With explicit swizzle: + %swizzled = cute.apply_swizzle(%ptr) { S<3,4,3> } : !llvm.ptr<3> + // Identity (no-op): + %swizzled = cute.apply_swizzle(%ptr) : !llvm.ptr<3> + ``` + + If the input pointer type has a non-identity swizzle attribute, this operation + computes the swizzled address by applying the swizzle transformation to the + pointer's address bits. The transformation is typically an XOR-based permutation + of certain address bits, as defined by the swizzle parameters. + + Swizzle is parameterized by three values: + - MBase: The number of least-significant bits to keep constant (not affected by the swizzle) + - BBits: The number of bits in the mask (the width of the swizzle region) + - SShift: The distance to shift the mask (positive values shift right, negative values shift left) + + The swizzle transformation operates on the address as follows: + + 0bxxxxxxxxxxxxxxxYYYxxxxxxxZZZxxxx + ^--^ MBase: least-significant bits kept constant + ^-^ ^-^ BBits: width of the mask (YYY and ZZZ) + ^---------^ SShift: shift distance for the YYY mask + + The swizzled address is computed by XOR'ing the ZZZ and YYY bitfields: + + Example: + + ``` + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxZZxxx + + the result is + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxAAxxx where AA = ZZ `xor` YY + ``` + + This operation is typically used just before dereferencing a pointer, + ensuring that the correct physical memory location is accessed according + to the swizzle pattern. If the pointer type has an identity swizzle, + this operation is a no-op and returns the original pointer. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrLLVMPointerType:$ptr, + OptionalAttr:$swizzle + ); + let results = (outs + Cute_CuTePointerTypeOrLLVMPointerType:$swizzled_ptr + ); +} +def Cute_AssumeOp : Cute_Op<"assume", [Pure]> { + let description = [{ + `cute.assume` represents assumptions such as divisibility for an SSA value. + unlike `llvm.assume`, `cute.assume` produces a strong typed value as output. + It is limited to support all constrains defined by `ConstrainedIntType`. + + Please see [constrained integer type](### ConstrainedIntType) for details. + + Example: + ```mlir + // Claim %1 is divisible by 8 + %1 = cute.assume(%0 : i32) : !cute.i32 + + ``` + + Note: + + // xxx: need to revisit + + ```mlir + %0 = ... -> !cute.i32 + if (%cond) { + %1 = cute.assume(%0 : !cute.i32) : !cute.i32 + ... // %1 has the assumption. + + // %0 is not assumed here + %data = load %ptr[%0] : ? + } + + // %0 here does not have any assumption. + ``` + }]; + let arguments = (ins + Cute_Integer:$src + ); + let results = (outs + Cute_IntegerTypeWithConstraints:$dst + ); +} +def Cute_BlockedProductOp : Cute_InferredOp<"blocked_product", [Pure]> { + let description = [{ + This operation performs "blocked_product" on the input and produce a blocked_product result, + and it automatically infer the "blocked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,2),(4,5)):((4,12),(1,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,?),(4,?)):((4,?{div=20}),(1,?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_CeilDivOp : Cute_InferredOp<"ceil_div", [Pure]> { + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_CoalesceOp : Cute_InferredOp<"coalesce", [Pure]> { + let summary = "`coalesce` op simplifies layout by combining modes that are possible to combine."; + let description = [{ + It does not respect the shape of the layout but preserve total size. + + ```mlir + // coalesce modes with contiguous/packed strides from left to right + %0 = cute.coalesce(%arg0) + : (!cute.layout<"((4,5),3):((1,4),20)">) -> !cute.layout<"60:1"> + + // coalesce from left to right but not right to left + %0 = cute.coalesce(%arg0) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">) -> !cute.layout<"(3,20):(20,1)"> + %0 = cute.coalesce(%arg0, %arg1) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">, !cute.coord<"(1,1)">) -> !cute.layout<"(3,20):(20,1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_ComplementOp : Cute_InferredOp<"complement", [Pure]> { + let summary = "Produces a disjoint layout."; + let description = [{ + The complement of a layout L is a layout L* such that (L,L*) achieves greater surjectivity. + That is, L* maps to elements in the codomain of L that are not in the image of L. + + It's worth to note that (L,L*) isn't alway surjective. + + Examples: + ``` + // trivial complement of L under it's domain is 1:0 + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:1">, !cute.shape<"3">) -> !cute.layout<"1:0"> + + // 3:2 is strided layout. complement of 3:2 under 6 is 2:1 filling stride + // (L,L*) => (3,2):(2,1) is a surjective layout + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:2">, !cute.shape<"6">) -> !cute.layout<"2:1"> + + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"(3,(5,2)):(1,(3,15))">, !cute.shape<"(6,20)">) -> !cute.layout<"4:30"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$cotarget + ); + let results = (outs + Cute_CuTeLayoutType:$res + ); +} +def Cute_ComposedGetInnerOp : Cute_InferredOp<"composed_get_inner", [Pure]> { + let summary = "Gets inner layout from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutTypeOrCuteSwizzleType:$result + ); +} +def Cute_ComposedGetOffsetOp : Cute_InferredOp<"composed_get_offset", [Pure]> { + let summary = "Gets offset from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_ComposedGetOuterOp : Cute_InferredOp<"composed_get_outer", [Pure]> { + let summary = "Gets layout outer from composed layout."; + let description = [{ + Example: + + ```mlir + %layout_outer = cute.composed_get_outer(%layout) + : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_CompositionOp : Cute_InferredOp<"composition", [Pure]> { + let description = [{ + The `composition` operation computes the composition of two layouts + (`!cute.layout`), in the following called `lhs` and `rhs`. + + Examples: + ``` + // Composition of two static layouts. + %0 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(5,4):(4,1)">) -> !cute.layout<"(5,4):(8,2)"> + + // Composition of a static and a dynamic layout. + %1 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(?,?):(?,?)">) -> !cute.layout<"(?,?):(?{div=2},?{div=2})"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$lhs, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$rhs + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$res + ); +} +def Cute_CopyAtomCallOp : Cute_Op<"copy_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.copy_atom_call` op executes a copy atom operation with the given operands."; + let description = [{ + Following src/dst layout of atom are valid: + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + Both `src` and `dst` operands are variadic, containing a variable number of tensors: + + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Examples: + + ```mlir + // Regular copy atom operation + cute.copy_atom_call(%copy_load, %src_v, %dst_v) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(((2,32),1)):(((1,131072),0))">, + !cute.memref, "((2,1)):((1,0))">) + -> () + ``` + + ```mlir + // TMA gather4: combine four 2D coordinate tensors into single destination + cute.copy_atom_call(%tma_gather4_atom, [%coord0, %coord1, %coord2, %coord3], %dst_v) + : (!cute_nvgpu.atom.tma_gather4<...>, + [!cute.coord_tensor<...>, !cute.coord_tensor<...>, + !cute.coord_tensor<...>, !cute.coord_tensor<...>], + !cute.memref) + -> () + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_CopyMakeFragmentOp : Cute_InferredOp<"copy.make_fragment", [Pure]> { + let description = [{ + Construct a copy fragment for the S or D side from either a partitioned view or an abstract TV + description. Intended for use between `cute.tiled.copy.partition_{S,D}` and `cute.copy`, but + also supports layout/shape inputs for RMEM/TMEM fragment construction similar to MMA. + + The op accepts: + 1. an operand id `S` or `D` (printed as a keyword); + 2. a copy atom value; + 3. either a partitioned view or a layout/shape to build the fragment. + + Returns a new ViewTypeInterface derived value to be consumed by `cute.copy`. + + Examples: + ```mlir + // Load from global memory (gmem -> rmem): build S and D fragments from S-partition + %atom_load = cute.make_atom() : () -> !cute_nvgpu.atom.universal_copy + %tiled_copy_load = cute.make_tiled_copy(%atom_load) + : !cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">> + %tGSrc_load = cute.tiled.copy.partition_S(%tiled_copy_load, %gSrc, %tid_coord) + : (!cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">>, + !cute.memref, "(128,128):(?{div=8},1)">, !cute.coord<"?">) + -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %frgS_load = cute.copy.make_fragment S (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)"> + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %rLoad = cute.copy.make_fragment D (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)" + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)" + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_CopyOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_CosizeOp : Cute_InferredOp<"cosize", [Pure]> { + let description = [{ + This operation calculates the cosize of a layout. + * For a layout `l`, cosize(l) = l(size(l) - 1) + 1. + * For a layout `l` with mode `mode`, cosize(l) = cosize(get(l)). + * For a composed layout 'l = a o b', cosize(l) = cosize(b). + + The `mode` attribute specifies the nested mode in the type for cosize calculation. + It indexes a sub-scope within the input type and returns the cosize of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the cosize of the entire type. + + ### Examples + + ```mlir + %input: !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + %0 = cute.cosize(%input) + : !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + // cosize(%input) = cosize((3,4):(4,1)), ret is 12 + ``` + + Example with mode: + ```mlir + %input: !cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))"> + %0 = cute.cosize(%input) <{mode=[1,0,1]}> + : (!cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))">) -> !cute.int_tuple<"109"> + // cosize(%input) = cosize((2,2):(36,72)), ret is 109 + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_Crd2IdxOp : Cute_InferredOp<"crd2idx", [Pure]> { + let description = [{ + Compute the index of a coordinate with respect to the given layout. I.e., + compute the natural coordinate of `coord` wrt. to the shape + and then compute the dot product with the strides. + + Examples: + ```mlir + // %crd0 is already a natural coordinate for %shape. + %0 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,(?,?))">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Compute the natural coordinate, then dot product. + %1 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Static and dynamic shape/strides are supported. + %3 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(4,2):(1,2)">) -> (i32) + // Strides with scaled basis produces a tuple. + %5 = cute.crd2idx(%crd, %layout) : (!cute.coord<"(?,?,?)">, !cute.layout<"(10,20,30):(1@0,1@1,20@1)">) -> (!cute.int_tuple<"(?,?)">) + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index + ); +} +def Cute_DereferenceArithTupleIteratorOp : Cute_InferredOp<"deref_arith_tuple_iter", [Pure]> { + let summary = "Dereferences an arithmetic tuple iterator to return the int_tuple value it holds."; + let description = [{ + Example: + + ```mlir + %0 = cute.deref_arith_tuple_iter(%arg0) : !cute.arith_tuple_iter<"(?,4)"> -> !cute.int_tuple<"(?,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$arith_tuple + ); +} +def Cute_DereferenceDescriptorIteratorOp : Cute_InferredOp<"deref_desc_iter", [Pure]> { + let description = [{ + This Operation dereferences a descriptor iterator to return the descriptor + value it holds. + }]; + let arguments = (ins + Cute_ATypeInterfaceForDescriptorIteratorTypes:$iter + ); + let results = (outs + AnyType:$value + ); +} +def Cute_DerefineOp : Cute_Op<"derefine", [Pure]> { + let description = [{ + The `cute.derefine` operation discard static information from modes + of a given cute algebra type like shape, stride, layout, coord or tile into + modes with less static information. + + For each mode, it either cast it into dynamic mode or keep it unchanged. In future, + it can be extended to support removal of `divby N` from dynamic mode. + + Casting dynamic mode into static mode is illegal. + + E.g. + + ```mlir + %dyn_0 = cute.derefine(%0) : !cute.shape<"(3,5)"> to !cute.shape<"(?,5)"> + %dyn_1 = cute.derefine(%1) : !cute.stride<"(?,5)"> to !cute.stride<"(?,?)"> + %dyn_2 = cute.derefine(%2) : !cute.coord<"(8,_)"> to !cute.coord<"(?,_)"> + ``` + }]; + let arguments = (ins + Cute_RefinableTypeInterfaceInstance:$src + ); + let results = (outs + Cute_RefinableTypeInterfaceInstanceOrInteger:$dst + ); +} +def Cute_DiceOp : Cute_InferredOp<"dice", [Pure]> { + let description = [{ + This operation dices a type implementing the `CuteValueTypeInterface` (ShapeType, IntTupleType, + LayoutType,...) using a static coordinate attribute. The profile of said coordinate must be + weakly congruent with that of `input`. Modes associated with an underscore are discarded in the + result. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_CoordAttr:$coord + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ElemLessOp : Cute_Op<"elem_less", [Pure]> { + let description = [{ + The cute.elem_less operation compares every elements in Lhs to the + corresponding elements in Rhs. The result is 1 if every Lhs elements + are less than corresponding Rhs elements, otherwise result is 0. + + `ElemLessOp` is useful when we want to know if a coordinate is out of bound: + + ```mlir + %is_in_bound = cute.elem_less(%coord, %shape) : !cute.coord<"(?,?)">, !cute.shape<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$lhs, + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_EqualOp : Cute_Op<"equal", [Pure]> { + let description = [{ + Checks two dynamic cute values for equality. Full static values are considered + equal only when their types are the same. + + Example: + + ```mlir + %1 = cute.make_shape : !cute.shape<"(32, 64)"> + %2 = cute.make_shape : !cute.shape<"(32, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(32,64)">, !cute.shape<"(32,64)"> + ``` + + The generated code only checks profile positions in which at least one side has a dynamic value. + Static comparisons are performed at compile time by the operation folder. + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(?, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(?,64)"> + ``` + + This would result in the first element being checked for equality in the runtime representation of the shapes. + + Note that this is dynamic if either of the operands will force that operand to be checked. So, + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(64, ?)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(64,?)"> + ``` + + will result in both dimensions being checked. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$lhs, + Cute_ValueTypeInterfaceInstance:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_FastDivmodComputeOp : Cute_Op<"fast_divmod.compute", [Pure]> { + let description = [{ + This operation performs a fast divmod using a fast divmod divisor. It computes the quotient and + the remainder without emitting division instructions. The quotient and the remainder are of the + same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient, + Cute_32BitSignlessIntegerOr64BitSignless:$remainder + ); +} +def Cute_FastDivmodCreateDivisorOp : Cute_Op<"fast_divmod.create_divisor", [Pure]> { + let description = [{ + This operation creates a fast divmod divisor from an integer. It is meant to be pre-computed, + ideally in host code, and then used throughout the lifetime of a device kernel for performing + divmods without emitting division instructions. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FastDivmodDivideOp : Cute_Op<"fast_divmod.divide", [Pure]> { + let description = [{ + This operation performs a fast divide using a fast divmod divisor. It computes the quotient + without emitting division instructions. The quotient is of the same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient + ); +} +def Cute_FastDivmodGetAuxOp : Cute_InferredOp<"fast_divmod.get_aux", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$shift1, + I8:$shift2 + ); +} +def Cute_FastDivmodGetDivisorOp : Cute_InferredOp<"fast_divmod.get_divisor", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); +} +def Cute_FastDivmodMakeDivisorOp : Cute_Op<"fast_divmod.make_divisor", [Pure]> { + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor, + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$sh1, + I8:$sh2 + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FilterOp : Cute_InferredOp<"filter", [Pure]> { + let summary = "This operation filters out dimensions with 0-strides and 1-sizes from a layout or view."; + let description = [{ + When working with tensor layouts, some dimensions may have stride 0 (indicating + broadcasting) or size 1 (singleton dimensions). This operation creates a new layout + or view with these dimensions removed, which can simplify subsequent operations. + + OptionalOperand `target_profile` is the user indicated filter control, with this operand, + the original layout's stride is ignored in the zero-stride selection. + But the `target_profile` will not take effect currently. This is a placeholder for future use. + + This operation accepts both layout and view types as valid input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_FilterZerosOp : Cute_InferredOp<"filter_zeros", [Pure]> { + let description = [{ + This operation produces a transformed layout or view by replacing the shape + of any mode with stride zero and non-unit extent with 1. This is commonly + applied prior to copy operations to prevent redundant or duplicated + data movement arising from broadcasted dimensions. + + The optional operand `target_profile` allows the user to specify a stride + filter profile; when provided, the selection of zero-stride dimensions is + determined by this stride profile rather than the original layout's stride + information. + + This operation accepts both layout and view types as input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$res + ); +} +def Cute_FlatDivideOp : Cute_InferredOp<"flat_divide", [Pure]> { + let summary = "This operation performs a flat_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_FlatProductOp : Cute_InferredOp<"flat_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_GetFlatCoordOp : Cute_InferredOp<"get_flat_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a flat coord. + + + The "flat" here means the reversed coordinate will be depth=1 coordinate. + This is useful when we just care about the rank semantic of the layout and don't care about the detail profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetHierCoordOp : Cute_InferredOp<"get_hier_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a hier coord. + + + The "hier" here means the reversed coordinate will have the same profile with the given layout. + This is useful when we care about both the rank semantic and the detail layout profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIntegralCoordOp : Cute_InferredOp<"get_integral_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a integral coord. + + + The "integral" here means the reversed coordinate will be a "scalar". + This is useful when we don't care about the layout's semantic and profile and we just want to know the offset in "domain". + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIterOp : Cute_InferredOp<"get_iter", [Pure]> { + let description = [{ + The "cute.get_iter" gets the underlying iterator from memref or coord_tensor. + It is solely as step during lowering, it has no side effects. + + Example: + + ```mlir + %ptr = cute.get_iter(%0) : !cute.memref, "((64,128),2):((1,1),64)"> + + // Or for TMA tensor + + %arith_tuple = cute.get_iter(%0) : !cute.coord_tensor<"(128, 64)", "((64,64),1):((1@0,1@0),64@1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$source + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$ptr + ); +} +def Cute_GetLayoutOp : Cute_InferredOp<"get_layout", [Pure]> { + let summary = "Gets the layout out of a `CuteViewTypeInterface`."; + let description = [{ + Example: + + ```mlir + %0 = cute.get_layout(%arg0) : !cute.memref + + %0 = cute.get_layout(%arg0) : !cute.memref o 0 o (256,?):(?,1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); +} +def Cute_GetLayoutsFromTileOp : Cute_InferredOp<"get_layouts_from_tile", [Pure]> { + let description = [{ + This operation extracts the layout elements out of a tile. A tile is a + layout, an underscore, or a tuple of tiles. + + Examples: + + - the type of the tile is !cute.tile<"[1:0;2:1]"> + 2 layouts are extracted of type !cute.layout<"1:0"> and !cute.layout<"2:1">, respectively + - the type of the tile is !cute.tile<[_;[1:0;2:1];3:1]> + 3 layouts are extracted of type !cute.layout<"1:0">, !cute.layout<"2:1">, and + !cute.layout<"3:1">, respectively + }]; + let arguments = (ins + Cute_CuteTileType:$tile + ); + let results = (outs + Variadic:$layouts + ); +} +def Cute_GetLeavesOp : Cute_InferredOp<"get_leaves", [Pure]> { + let description = [{ + The `cute.get_leaves` operation extracts all leaf values from a hierarchical tuple into + a flat sequence of results. Each result corresponds to a leaf value found in the input tuple, + preserving the order of a depth-first traversal. + + The operation handles nested tuples of any depth, including tuples containing other tuples. + The type of each result matches the type of the corresponding leaf value in the input. + + Example: + ```mlir + // Extract leaves from a hierarchical shape tuple + %0:3 = cute.get_leaves(%hier) : !cute.shape<"(3,?{div=4},5)"> + -> (!cute.shape<"3">, !cute.shape<"?{div=4}">, !cute.shape<"5">) + + // Extract leaves from a nested int tuple + %1:4 = cute.get_leaves(%t) : !cute.int_tuple<"((1,2),(3,4))"> + -> (!cute.int_tuple<"1">, !cute.int_tuple<"2">, + !cute.int_tuple<"3">, !cute.int_tuple<"4">) + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$input + ); + let results = (outs + Variadic:$results_ + ); +} +def Cute_GetOp : Cute_Op<"get", [Pure]> { + let description = [{ + This operation returns a potentially nested mode out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a tuple, a layout,... + + The optional `mode` attribute indices are used to select the portion of the + input type. The number of indices in the `mode` attribute should be less + than or equal to the depth of type. If `mode` is empty, a copy of the input + value is returned. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.get(%shape) <{mode=[1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"(2,4)"> + %mode1 = cute.get(%shape) <{mode=[1,1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"4"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + OptionalAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr:$result + ); +} +def Cute_GetScalarsOp : Cute_InferredOp<"get_scalars", [Pure]> { + let summary = "Gets the scalars out of a value with CuteValueTypeInterface."; + let description = [{ + This operation returns only the dynamic scalars when `only_dynamic` is set. + + A "scalar" in this context refers to a single value used as part of the + state of a type. In general, each state value held by a type will be + considered a scalar if it is printed in the text format. This includes: + - static integers + - dynamic integers + - swizzle parameters + - scaled basis value (static integer, ratio (num + denom), dynamic integer) + - scaled basis modes + - indexed pointer address (always a dynamic value) + - indexed pointer stride (dynamic integer or dynamic) + + Note that underscore values in coordinate types are NOT considered scalars. + + Example: + + ```mlir + %1 = cute.get_shape(%0) : (!cute.layout<"(256,?):(?,1)">) -> !cute.shape<"(256,?)"> + %2 = cute.get_scalars(%1) <{only_dynamic}> : (!cute.shape<"(256,?)">) -> i32 + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints:$cute_value, + UnitAttr:$only_dynamic + ); + let results = (outs + Variadic:$scalars + ); +} +def Cute_GetShapeOp : Cute_InferredOp<"get_shape", [Pure]> { + let summary = "get shape from view, layout or tile"; + let description = [{ + Example: + + ```mlir + %shape = cute.get_shape(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.shape<"(32,64)"> + %shape = cute.get_shape(%tile) + : (!cute.tile<"[(32,64):(1,32);3:1]>">) -> !cute.shape<"((32,64),3)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2:$input + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_GetStrideOp : Cute_InferredOp<"get_stride", [Pure]> { + let summary = "get stride from layout."; + let description = [{ + Please be noted that stride is not generalized for view and layout because composed layout + doesn't have generalized layout concept. + + Example: + + ```mlir + %stride = cute.get_stride(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.stride<"(1, 32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_GroupModesOp : Cute_InferredOp<"group_modes", [Pure]> { + let description = [{ + This operation will make the input view/layout's [B,E) modes + into one new mode with one-depth deeper. + + Example: + + ```mlir + %res = cute.group_modes(%arg0) <0,1> + : (!cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)">) + -> !cute.coord_tensor<"(0,0)", "((512),64,2):((1@1),1@0,64@0)"> + %res = cute.group_modes(%arg0) <0,2> + : (!cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o (9,9,9):(1@1,1@0,1@2)">) + -> !cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o ((9,9),9):((1@1,1@0),1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + I32Attr:$begin, + I32Attr:$end + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_Idx2CrdOp : Cute_InferredOp<"idx2crd", [Pure]> { + let description = [{ + compute the coordinate corresponding to an index with respect to the given shape. I.e., + compute the coordinate that maps to the given index using given shape and default stride. + Note: This only works for compact layout. + + Examples: + ```mlir + // Convert a linear index to coordinate using shape and default stride. + %0 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(4,2)">) -> (!cute.coord<"(?,?)">) + // Static and dynamic shape are supported. + %1 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(?,(?,?))">) -> (!cute.coord<"(?,(?,?))">) + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$coord + ); +} +def Cute_IncrementCoordOp : Cute_InferredOp<"increment_coord", [Pure]> { + let summary = "This operation colexicographically increments a coord within a coord space defined by a shape."; + let description = [{ + Example: + + ```mlir + // With static coord and shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(0,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(1,0)"> + %coord2 = cute.increment_coord(%coord1, %shape) : + (!cute.coord<"(1,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(0,1)"> + + // With dynamic coord or shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(?,?)">, !cute.shape<"(?,8)">) -> !cute.coord<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_IntToPtrOp : Cute_Op<"inttoptr", [Pure]> { + let description = [{ + The `cute.inttoptr` operation accepts an integer value and cast to cute + pointer value with align=1 (byte) by default. + + To create ptr with align > 1, we must use constrained int type with divby > 1. + Alignment of result ptr is the same as `divby` of constrained int type + + Example: + + ```mlir + // Convert to cute.ptr with unrestricted integer type. Can only have align<1> + %ptr = cute.inttoptr(%0) : i64 to !cute.ptr> + + // Convert to cute.ptr with constrained integer type. + %aligned_ptr = cute.inttoptr(%0) : !cute.i64 to !cute.ptr> + ``` + }]; + let arguments = (ins + Cute_IntegerOrIntegerTypeWithConstraints:$src + ); + let results = (outs + Cute_PointerTypeInterfaceInstance:$dst + ); +} +def Cute_KernelSmemSizeOp : Cute_Op<"kernel_smem_size", [Pure]> { + let description = [{ + This host-side operation gets the static smem size from a kernel function + and returns the size as an i64 integer. + }]; + let arguments = (ins + SymbolRefAttr:$kernel_name + ); + let results = (outs + I64:$size + ); +} +def Cute_LeftInverseOp : Cute_InferredOp<"left_inverse", [Pure]> { + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_LoadScaledIndexOp : Cute_InferredOp<"load_scaled_index", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `cute.load_scaled_index` operation adds the offset (%ptr_b[%index] * %stride) to a + cute pointer %ptr_a. %ptr_a can be any cute pointer type, while %ptr_b must be an I32 cute pointer. + The behavior is undefined if %ptr_b[%index] results in an out-of-bound access. + The result is a cute pointer that has identical type as %ptr_a except that its alignment is derived + from ptr_a's alignment and the stride. + + Example: + + ```mlir + %ptr = cute.load_scaled_index(%src_ptr, %table_ptr, %index, %stride) + : (!cute.ptr, !cute.ptr, !cute.int_tuple<"?">, !cute.int_tuple<"?{div=16}">) + -> !cute.ptr + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$ptr_a, + Cute_CuTePointerTo32BitSignlessInteger:$ptr_b, + Cute_IntTupleTypeThatDepthIs0:$index, + Cute_IntTupleTypeThatDepthIs0:$stride + ); + let results = (outs + Cute_CuTePointerType:$result + ); +} +def Cute_LocalPartitionOp : Cute_InferredOp<"local_partition", [Pure]> { + let description = [{ + The `local_partition` operation tiles a tensor according to the flat shape of a layout + that provides the coordinate of the target index. + This is typical at the Thread level where data is partitioned across repeated patterns + of threads. + + Example: + + ```mlir + %local_partition = cute.local_partition(%input, %tile, %index) + : (!cute.memref, "(16,?):(1,16)">, !cute.layout<"(2,4)">, i32) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_StaticLayout:$tiler, + Cute_32BitSignlessIntegerOr64BitSignless:$index + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LocalTileOp : Cute_InferredOp<"local_tile", [Pure]> { + let description = [{ + The `local_tile` operation tiles a tensor according to a tiler and use a coord to + index into the remainder, keeping the tile. + + This is typical at the CTA level where tiles of data are extracted: + + ```mlir + %data = // ( M, N) + %crd = cute.make_coord(blockIdx.x,blockIdx.y) : !cute.coord<"(?,?)"> + %cta_data = cute.local_tile(%data, %tile, %crd) : ... // (_32,_64) + ``` + + Example: + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_,_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(0,_,_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1):(1@1,1@0,0,1@2)"> + ``` + + Utilizing a flat coordinate '(_)' will flatten the resulting tile. + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1,1):(1@1,1@0,0,0,1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tile, + Cute_CuteCoordType:$coord, + OptionalAttr:$proj + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LogicalDivideOp : Cute_InferredOp<"logical_divide", [Pure]> { + let description = [{ + This operation performs "logical_divide" on the input and produces a logical_divided result, + and it automatically infers the "logical_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.logical_divide(%alloc, %shape) + : (!cute.memref, !cute.shape<"(4,8)">) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_LogicalProductOp : Cute_InferredOp<"logical_product", [Pure]> { + let description = [{ + This operation performs "logical_product" on the input and produce a logical_product result, + and it automatically infer the "logical_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.logical_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,4),(2,5)):((4,1),(12,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.logical_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,4),(?,?)):((4,1),(?{div=20},?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_MakeArithTupleIteratorOp : Cute_Op<"make_arith_tuple_iter", [Pure]> { + let description = [{ + This Operation makes an arithmetic tuple iterator (`!cute.arith_tuple_iter<...>`) + from an int_tuple value (`!cute.int_tuple<...>`) if provided, or it is empty + when all the modes are static. + + Its verifier checks that the value type of the result iterator matches the + operand value type if provided. + }]; + let arguments = (ins + Optional:$value + ); + let results = (outs + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); +} +def Cute_MakeAtomOp : Cute_Op<"make_atom", [Pure]> { + let summary = "`cute.make_atom` op is generic constructor of SSA value of copy or mma atom type."; + let description = [{ + It optionally takes variadic list of SSA values as well as named attribute to + initialize corresponding fields of copy or mma atom. + + Examples: + + ```mlir + // create copy atom which doesn't require initialization + %0 = cute.make_atom() : () + -> !cute_nvgpu.atom.tmem_load + + // create mma atom and initialize it with SSA values + %1 = cute.make_atom(%accum_c, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<128x128x16, + num_cta = 1, + ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0> + ``` + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_AtomTypeInterfaceInstance:$atom + ); +} +def Cute_MakeComposedLayoutOp : Cute_InferredOp<"make_composed_layout", [Pure]> { + let summary = "The `make_composed_layout` operation constructs composed layout with"; + let description = [{ + * (inner) affined layout or swizzle layout + * offset + * (outer) layout + + In CuTe algebra, composed layout represent composition of + + ```math + L_{inner} o offset o L_{outer} + ``` + + where outer layout defined the domain of the layout which is visible to user. For given input coordinate + in domain of outer layout, composed layout maps coordinate to codomain by + + ```math + L_{inner}(L_{outer}(coordinate) + offset) + ``` + + Composed Layout is useful in cases where composition() does not or cannot apply to LayoutA and LayoutB. + For example, when the "divisibility condition" in shape_div is violated in composition(LayoutA, LayoutB). + + This ComposedLayout provides similar functionality to Layout including tiling, partitioning, + coordinate-to-index mapping and layout manipulations, but is not considered a "normal" layout. + For example, this layout provides shape() and size() functions, but does not provide stride() functions. + Mostly, the similar functionality is accomplished by applying each operation to LayoutB only + as LayoutB defines the domain. + + ### Examples: + + ```mlir + // Swizzle composed layout + %inner : !cute.swizzle<"S<3,4,3>"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"S<3,4,3> o 5 o (?,?):(?,?)"> + + // Linearized composed layout + %inner : !cute.layout<"((?,?),(?,?)):((?,?),(?,?))"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"((?,?),(?,?)):((?,?),(?,?)) o ? o (1,2):(3,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutTypeOrSwizzleWithValidParameters:$inner, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset, + Cute_CuTeLayoutType:$outer + ); + let results = (outs + Cute_CuTeComposedLayoutType:$result + ); +} +def Cute_MakeCoordOp : Cute_Op<"make_coord", [Pure]> { + let description = [{ + Constructs a coord. The operands correspond to the + coord's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting coord type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_coord(%0, %1) : (i32, i32) + -> !cute.coord<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_MakeDescriptorIteratorOp : Cute_Op<"make_desc_iter", [Pure]> { + let description = [{ + This Operation makes a descriptor iterator from a descriptor value. Its + verifier verifies that the value Type of the result Type implementing the + iterator Type Interface is the same as the Type as the input operand. + }]; + let arguments = (ins + Cute_ATypeInterfaceForMemoryDescriptorsSuchAs:$value + ); + let results = (outs + AnyType:$iter + ); +} +def Cute_MakeFragmentLikeOp : Cute_InferredOp<"make_fragment_like", [Pure]> { + let summary = "A deprecated operation but used by collective DSL (keep it here to avoid breaking changes)"; + let description = [{ + TODO: remove this operation in the future + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeIdentityLayoutOp : Cute_InferredOp<"make_identity_layout", [Pure]> { + let summary = "This operation makes an identity layout with the given shape."; + let description = [{ + Example: + ```mlir + %sha = cute.make_shape() : () -> !cute.shape<"(2,(3,4),5)"> + %lay = cute.make_identity_layout(%sha) : !cute.layout<"(2,(3,4),5):(1@0,(1@0@1,1@1@1),1@2)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_MakeIdentityTensorOp : Cute_InferredOp<"make_identity_tensor", [Pure]> { + let description = [{ + The cute.make_identity_tensor operation construct an identity_tensor + based on the given shape. + + Example to make an identity_tensor from a static shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"(0,(0,0))", "(2,(3,4)):(1@0,(1@0@1,1@1@1))"> + ``` + + Example to make an identity_tensor from a dynamic shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"((0,0),(0,0))", "((?,?),(?,?)):((1@0@0,1@1@0),(1@0@1,1@1@1))"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordTensorType:$result + ); +} +def Cute_MakeIntTupleOp : Cute_Op<"make_int_tuple", [Pure]> { + let description = [{ + Constructs a int_tuple. The operands correspond to the + int_tuple's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting int_tuple type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_int_tuple(%0, %1) : (i32, i32) + -> !cute.int_tuple<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_MakeLayoutLikeOp : Cute_InferredOp<"make_layout_like", [Pure]> { + let description = [{ + Creates a compact memory layout that: + 1. Has a congruent shape with the source layout + 2. Has strides in the same order of major-ness as the input layout + 3. Mode 0 strides are `LayoutLeft` (column major) with zero strides preserved from the input layout + + Mode-0 constraints are applied due to its common usage with MMA_Atoms and Copy_Atoms + that need to create RMEM fragments. + + Example: + ```mlir + %0 = cute.make_layout_like(%layout) + : !cute.layout<"(4,8):(1,1024)"> to !cute.layout<"(4,8):(1,4)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeLayoutOp : Cute_Op<"make_layout", [AttrSizedOperandSegments, Pure]> { + let description = [{ + The `make_layout` operation generates a layout based on provided shape and stride. + If the stride component is omitted in the operation's result type + (e.g., `!cute.layout<"(?,(?,?))">`), the operation defaults to a column-major + layout, and missing stride details are automatically inferred to complete the type + of the resulting layout. + + The shape or stride operand must be supplied if they contain dynamic elements. + However, if the shape or stride is entirely static, corresponding operands can be + neglected. + + When the result type does not specify stride information (like `!cute.layout<"(?,?)">` + ), it is not necessary to provide a stride operand, even if the stride to be inferred + includes dynamic components (as in the inferred type `!cute.layout<"(?,?):(1,?)">`). + + ### Examples: + + ```mlir + // Shape and stride are both dynamic + %layout = cute.make_layout(%shape, %stride) : !cute.layout<"(?,64):(1,?)"> + + // Shape and stride are both static + %layout = cute.make_layout() : !cute.layout<"(32,64):(1,32)"> + + // Stride is omitted in the result type, implying it should be inferred + %layout = cute.make_layout(%shape) : !cute.layout<"(?,(?,?))"> + ``` + }]; + let arguments = (ins + Optional:$shape, + Optional:$stride + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeOrderedLayoutOp : Cute_InferredOp<"make_ordered_layout", [Pure]> { + let description = [{ + The `make_ordered_layout` operation generates a compact layout based on provided `shape` + and stride is calculated by `shape` and `order`. + + Different from `make_layout`, `shape` operand is required, `order` is optional. When `order` + isn't set, default stride growth order is from left to right. Supports dynamic values in `shape`, + while currently `order` is restricted to static. Rank of `order` should be equal to rank of + shape but no need for congruent. + + Note that numbers in `order` are just used for relative order relationship, e.g. (67,46,23,32) + is equivalent to (4,3,1,2), (3,2,4) is equivalent to (2,1,3). + + ### Examples: + + ```mlir + %shape = cute.make_shape() : !cute.shape<"(32,64)"> + %shape_dyn = cute.make_shape() : !cute.shape<"(32,?)"> + %order = cute.make_shape() : !cute.shape<"(1,2)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,64):(1,32)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,?):(1,32)"> + + %order = cute.make_shape() : !cute.shape<"(2,1)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,64):(64,1)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,?):(?,1)"> + + %shape = cute.make_shape() : !cute.shape<"(2,(2,2),2)"> + %order = cute.make_shape() : !cute.shape<"(0,(2,3),1)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(1,(4,8),2)"> + + %order = cute.make_shape() : !cute.shape<"(2,1,3)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(4,(1,2),8)"> + }]; + let arguments = (ins + Cute_CuteShapeType:$shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$order + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeShapeOp : Cute_Op<"make_shape", [Pure]> { + let description = [{ + Constructs a shape. The operands correspond to the + shape's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting shape type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_shape(%0, %1) : (i32, i32) + -> !cute.shape<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_MakeSparseElemOp : Cute_Op<"make_sparse_elem", [Pure]> { + let summary = "Creates a sparse element from a physical storage."; + let description = [{ + Example: + + ```mlir + %v0 = arith.constant 8 : i8 + %v1 = arith.constant 31 : i8 + %physical_storage = vector.from_elements(vector<2xf16>, %v0, %v1) + %sparse_elem = cute.make_sparse_elem(%physical_storage) : (!cute.sparse_elem<4, vector<2xf16>>) + ``` + }]; + let arguments = (ins + AnyType:$physical_storage + ); + let results = (outs + Cute_CuTeSparseElementType:$result + ); +} +def Cute_MakeStrideOp : Cute_Op<"make_stride", [Pure]> { + let description = [{ + Constructs a stride. The operands correspond to the + stride's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting stride type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_stride(%0, %1) : (i32, i32) + -> !cute.stride<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_MakeTileOp : Cute_Op<"make_tile", [Pure]> { + let description = [{ + Constructs a tile. The operands correspond to the + tile's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting tile type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_tile(%0, %1) : (i32, i32) + -> !cute.tile<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteTileType:$result + ); +} +def Cute_MakeTiledCopyOp : Cute_Op<"make_tiled_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a "base" copy atom + instance. Copy atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute_nvgpu.arch` operations. + + Example: + The below values are origin from cutlass SM100_f16f16_UMMA kernel, the original code: + auto tAgA_mkl = cta_tma_a.partition_S(cta_mma.partition_A(gA_mkl))(_,_0{},_0{},_0{},_,_,_); + All the values are from the "cta_tma_a". + ```mlir + %0 = cute.make_atom(%tma_desc_ptr, %tma_bar) : (!cute.ptr>!cute.ptr>) + -> !cute_nvgpu.atom.tma_load + %1 = cute.make_tiled_copy(%0) + : !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledCopyAtomType:$result + ); +} +def Cute_MakeTiledCopyV2Op : Cute_Op<"make_tiled_copy_v2", [Pure]> { + let description = [{ + This operation creates a V2 tiled copy instance by tiling a V2 copy atom. + V2 copy atoms use fragment attributes for operand-specific behavior. + + Example: + ```mlir + %0 = cute.make_atom() : () -> !cute_nvgpu.atom.sm100_s2t_copy_v2 + %1 = cute.make_tiled_copy_v2(%0) + : !cute.tiled_copy_v2, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceV2Instance:$atom + ); + let results = (outs + Cute_TiledCopyV2AtomType:$result + ); +} +def Cute_MakeTiledMmaOp : Cute_Op<"make_tiled_mma", [Pure]> { + let description = [{ + This operation creates a new MMA atom instance by tiling a "base" MMA atom + instance. MMA atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute.arch` operations. + + Example: + ```mlir + %0 = cute.make_atom(%accum, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<64x64x16, num_cta = 1, ab_major = (k,mn), elem_type = (f16,f16,f16), frag_kind = ss, c_scale_exp = 0> + %1 = cute.make_tiled_mma(%0) + : !cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledMMAAtomType:$result + ); +} +def Cute_MakeTupleOp : Cute_Op<"make_tuple", [Pure]> { + let description = [{ + Creates a tuple from multiple values. The number and types of the + parameters must match the number and types of the result tuple. + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_ATupleIsAFixedLengthOrderedList:$result + ); +} +def Cute_MakeViewOp : Cute_Op<"make_view", [Pure]> { + let description = [{ + The cute.make_view operation construct `CuteViewTypeInterface` + (i.e., cute.memref or cute.coord_tensor) with given `iter` and `layout`. + Internally, we infer the exact return type by checking `iter`'s type: + - if `iter` is cute.ptr, return type is `CuteMemRefType` + - if `iter` is cute.tuple, return type is `CuteCoordTensorType` + + `iter` and `layout` could contain both static and dynamic values. + + `MakeViewOp` usually is the result of a previous operation + (e.g., cute.slice, cute.zipped_divide), where creating a new + `CuteViewTypeInterface` is needed. + + `MakeViewOp` does not involve allocating buffers of any kind. + + Example to make cute.coord_tensor with an iter of cute.tuple type: + + ```mlir + %coord_tensor= cute.make_view(%iter, %layout) : !cute.coord_tensor<"(0,2)", "(30):(20@1)"> + ``` + + Example to make cute.memref with an iter of cute.ptr type: + + ```mlir + %memref= cute.make_view(%iter, %layout) : !cute.memref + ``` + + Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref + ``` + + Composed+Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref o 9 o (64):(16)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$iter, + Optional:$layout + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_MemRefAllocSmemOp : Cute_Op<"memref.alloc_smem"> { + let description = [{ + The memref.alloc_smem operation is utilized for the allocation of static + shared memory. The cosize of the memref layout, which must be static, + determines the size of the memory allocated by this operation. + + ```mlir + %0 = cute.memref.alloc_smem() : !cute.memref + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MemrefInSmemWithStaticAndNonComposed:$memref + ); +} +def Cute_MemRefAllocaOp : Cute_Op<"memref.alloca"> { + let description = [{ + The memref.alloca operation is used to allocate shared/stack memory, which is automatically + released when control returns from the closest surrounding operation's region. This + operation only supports memref with the address space reg, and attempting to allocate + memory with other address spaces will result in an error. + + The size of memory allocated is determined by the layout operand's cosize, which can + be either static or dynamic. The layout operand can be omitted when the memref layout + is static. + + If the memref is sparse, the allocated memory size will be layout operand's cosize + divided by sparsity factor. + + ```mlir + // %dynamic_layout : !cute.layout<"(?,24)"> + %0 = cute.memref.alloca(%dynamic_layout) : !cute.memref + // static layout + %1 = cute.memref.alloca() : !cute.memref + ``` + }]; + let arguments = (ins + Optional:$layout + ); + let results = (outs + Cute_MemrefInGenericOrRegisterMemOrSmem:$result + ); +} +def Cute_MemRefLoadOp : Cute_InferredOp<"memref.load", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load` operation loads one element at the specified logical coordinate from a memref + in gmem/rmem/smem/generic memory spaces to a register. This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. For CuteMemRefType, applies swizzling if needed, converts to LLVM pointer, and performs the load + 5. For CoordTensorType, returns the tuple directly + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treat it as f32 even bitwidth of MLIR type is 19. + + Example: + + ```mlir + // Load a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + %0 = cute.memref.load(%memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> f32 + + // Load a tf32 value (will be handled with special bitwise casting) + %1 = cute.memref.load(%tf32_memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> tf32 + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor2:$src, + Cute_CuteCoordType:$coord + ); + let results = (outs + AnyType:$result + ); +} +def Cute_MemRefLoadVecOp : Cute_InferredOp<"memref.load_vec", [AttrSizedOperandSegments, MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load_vec` operation loads all elements in the logical coordinate + space from a memref generic or register memory to a 1D vector. Only memrefs with + static layouts are accepted. + + The resulting vector has the same element type as the memref, with a size + equal to the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the output vector) can be + provided to mask out elements. When the mask is present, an optional + pass_thru vector can also be provided to fill in values for elements that + are masked out, and thus the result is equivalent to a regular memref load + followed by a select between the loaded and the passthru values, predicated + on the same mask, except that the masked-off lanes are not accessed. + + Example: + + ```mlir + // Load vector + %0 = cute.memref.load_vec(%memref) : (!cute.memref) -> vector<32xf32> + + // Load vector from swizzled memory + %0 = cute.memref.load_vec %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Load vector with mask and pass_thru + %0 = cute.memref.load_vec(%memref, %mask, %pass_thru) + : (!cute.memref, vector<32xi1>, vector<32xf32>) -> vector<32xf32> + ``` + }]; + let arguments = (ins + Cute_StaticShape:$src, + Optional:$mask, + Optional:$pass_thru + ); + let results = (outs + Cute_OfRanks1:$result + ); +} +def Cute_MemRefStoreOp : Cute_Op<"memref.store"> { + let description = [{ + The `memref.store` operation stores a register value to a memref in gmem/rmem/smem/generic at given logical coordinate. + This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. Converts the pointer to the appropriate LLVM pointer type and performs the store + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treats it as f32 even though the bitwidth of the MLIR type is 19. + This special handling allows for proper conversion between tf32 and f32 representations during store operations. + + Example: + + ```mlir + // Store a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + cute.memref.store(%memref, %coord, %value) : (!cute.memref, !cute.coord<"(?,?)">, f32) + + // Store a tf32 value (will be handled with special bitwise casting) + cute.memref.store(%tf32_memref, %coord, %tf32_value) : (!cute.memref, !cute.coord<"(?,?)">, tf32) + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor3:$dst, + Cute_CuteCoordType:$coord, + AnyType:$value + ); + let results = (outs + ); +} +def Cute_MemRefStoreVecOp : Cute_Op<"memref.store_vec"> { + let description = [{ + The `memref.store_vec` operation stores all elements of a 1D vector to a to the logical coordinate + space of memref in generic or register memory. Only memrefs with static layouts are accepted. + + The vector has the same element type as the memref, with a size equal to + the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the input vector) can be + provided to mask out elements. + + ```mlir + // Store vector + %0 = cute.memref.store_vec(%value, %memref) : (vector<32xf32>, !cute.memref) -> () + + // Store vector to swizzled memory + %0 = cute.memref.store_vec %value, %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Store vector with mask + %0 = cute.memref.store_vec(%value, %memref, %mask) + : (vector<32xf32>, !cute.memref, vector<32xi1>) -> () + ``` + }]; + let arguments = (ins + Cute_OfRanks1:$value, + Cute_StaticShape:$dst, + Optional:$mask + ); + let results = (outs + ); +} +def Cute_MmaAtomCallOp : Cute_Op<"mma_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.mma_atom_call` op executes a mma atom operation with the given operands."; + let description = [{ + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular MMA, `a` and `b` contain the MMA A and B tensors respectively. + - For MMA with auxiliary operands, `a` and `b` contain the MMA A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledMMA, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseMMA, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseMMA, `a` = [A, SFA, E] and `b` = [B, SFB]. + Example (regular MMA): + + ```mlir + // Call a mma atom operation + cute.mma_atom_call(%mma_atom, %d, %a, %b, %c) + : (!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0>, + !cute.memref, "((128,128)):((131072,1))">, + !cute_nvgpu.smem_desc_view, + !cute_nvgpu.smem_desc_view, + !cute.memref, "((128,128)):((131072,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_MmaMakeFragmentOp : Cute_InferredOp<"mma.make_fragment", [Pure]> { + let description = [{ + Given a partitioned tensor, the `cute.mma.make_fragment` operation returns + a tensor with the correct memory space for the specified operand and dot + atom. This op is usually used between partition and gemm. It is the + "prepare" step before gemm. + + `cute.tiled.mma.partition` returns a view into the specified tensor, but it + does not copy the data to the correct memory space. This is done by + `cute.mma.make_fragment` and explicit memcpy operations. Whether a memcpy + is needed or not depend on the result memory space. + + * RMEM: The result tensor is a newly allocated tensor with unspecified + contents. A memcpy is needed. + * SMEM: The result tensor is a view into the input tensor in the form of a + CuteSmemDescViewType. A memcpy is not needed. + * TMEM: The result tensor uses a dummy TMEM engine (fake pointer 0x0) and + must be set correctly later, aligned with Cute C++. + + Instead of a tensor, this op also accepts cute.layout (for RMEM/TMEM) and + cute.shape (for RMEM/TMEM). + + Example: + + ```mlir + // Example 1: Full example for SM80. + // Partition a tensor for operand A, which is stored in SMEM. + %partitioned = cute.tiled.mma.partition A(%atom, %smem_A, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // The tensor is still in SMEM, but sm80.mma expects A in registers. + // Prepare a tensor with the correct shape and strides in RMEM. + %fragment = cute.mma.make_fragment A (%3, %partitioned) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)">) + -> !cute.memref, "((2,2,2),4,2):((1,2,4),16,8)"> + + // Copy the data from SMEM to RMEM. + cute.copy(..., %partitioned, %fragment) + + // Example 2: Make fragment for SM100 (SMEM). + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "((128,16),2,32):((512,1),65536,16)">) + -> !cute_nvgpu.smem_desc_view + + // Example 3: Make fragment for SM100 (TMEM) from layout. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.layout<"((64,16),1,4):((1,64),0,1024)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + + // Example 3: Make fragment for SM100 (TMEM) from shape. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.shape<"((64,16),1,4)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_PrefetchAtomCallOp : Cute_Op<"prefetch_atom_call", [MemoryEffects<[MemRead]>]> { + let summary = "`cute.exec_prefetch_atom` op executes a prefetch atom operation with the given operands."; + let description = [{ + Following src layout of atom are valid + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`. + + Examples: + + ```mlir + // Call a prefetch atom operation + cute.prefetch_atom_call(%copy_tma_load, %src_v) + : (!cute_nvgpu.atom.tma_load, + %src: !cute.coord_tensor<"(0,0)", "(((64,128),1),1,1,2):(((1@0,1@1),0),0,0,64@0)">) + -> () + + // gather4 prefetch: data tensor + gmem index tensor + cute.prefetch_atom_call(%gather4_atom, %src_v, %idx_v) + : (!cute_nvgpu.atom.tma_load>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + -> () + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_PrependToRankOp : Cute_InferredOp<"prepend_to_rank", [Pure]> { + let summary = "Prepend element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(3,4,5)"> + %elem = cute.make_shape() : !cute.shape<"(1,2)"> + %new_layout = cute.prepend_to_rank<5> (%shp, %elem) + : (!cute.shape<"(3,4,5)">, !cute.shape<"1">) -> !cute.shape<"((1,2),(1,2),3,4,5)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_PrintOp : Cute_Op<"print"> { + let summary = "Debug operation to print out value of cute data type."; + let description = [{ + Supported following cute data types + * shape, stride, layout, tile + * View: memref, coord_tensor + * Iterator: ptr, tuple + + Examples: + + ```mlir + cute.print(%0) : !cute.shape<"(3,5)"> + + cute.print(%1) : !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // variadic operands without format string + cute.print(%0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // with format string + cute.print("{} -> {}", %0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + ``` + + The optional `operand_signed` attribute carries the DSL-level signedness of + each operand, one entry per value. It exists because DSL integers lower to + signless MLIR integer types, so an unsigned value is indistinguishable from + a signed one by type alone. When an entry is false, a generic `{}` + placeholder for that integer operand is rendered with an unsigned specifier + (`%u`/`%llu`) instead of `%d`/`%lld`. When absent, all operands are treated + as signed, and an explicit `%u` in the format string still forces unsigned. + }]; + let arguments = (ins + Variadic:$values, + OptionalAttr:$fmt, + OptionalAttr:$operand_signed + ); + let results = (outs + ); +} +def Cute_PrintTMADescIm2ColOp : Cute_Op<"print_tma_desc_im2col"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{host = true}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_im2col(%tma_desc_a_host_ptr_i64, %fd) + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintTMADescTiledOp : Cute_Op<"print_tma_desc_tiled"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %smem_ablayout_128_64_k_major, %cta_v_tile_128_64_k_major) + <{mode = host = true}> : + ( + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.composed_layout<"S<3, 4, 3> o 0 o ((128,64),1,1):((64,1),0,0)">, + !cute.layout<"((128,64),1,1):((1@0,1@1),0,0)"> + ) -> !cute.ptr + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_tiled(%tma_desc_a_host_ptr_i64, %fd) + ``` + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintViewOp : Cute_Op<"print_view"> { + let summary = "Debug operation to print cute view types."; + let description = [{ + Currently supports cute.memref and cute.tensor. MemRefs with both static layout + and dynamic layout are accepted. An optional coord parameter can be used to + specify the element to be printed. If no coord is provided, all elements will + be printed. + + The `verbose` attribute controls the output format: + - When `verbose` is true (default), it prints the full details of the view + including memory address, shape, and strides. + - When `verbose` is false, it prints in a more compact torch-style matrix format. + + The `signed` attribute is used for integer data types to control whether values + should be interpreted as signed or unsigned. This attribute is ignored for + floating point types and coord tensors. + + Examples: + + ```mlir + cute.print_view(%0) : !cute.memref + output: + Memref raw_ptr(0x0000152397fffc80: f32, rmem, align<4>) o (2,2):(2,1) = ( + (0,0)=1.000000 + (0,1)=2.000000 + (1,0)=3.000000 + (1,1)=4.000000 + ) + + cute.print_view(%1) : !cute.memref + output: + Memref raw_ptr(0x0000148eabfffbf0: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)) = ( + (0,(0,0))=1.000000 + (0,(0,1))=2.000000 + (0,(1,0))=3.000000 + (0,(1,1))=4.000000 + (1,(0,0))=5.000000 + (1,(0,1))=6.000000 + (1,(1,0))=7.000000 + (1,(1,1))=8.000000 + ) + + cute.print_view(%3,%coord) : !cute.memref,!cute.coord<"(?,(?,?))"> + output: + Memref raw_ptr(0x0000150ce3fffc90: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)), coord (1,(1,0)) = (7.000000) + + // Using verbose=false to print in torch-style format + cute.print_view(%0) {verbose = false} : !cute.memref + + output: + tensor(raw_ptr(0x000000003746a2e0: f64, generic, align<8>) o (8,5,3):(15,3,1), data= + [[[ 1.426091, -0.279034, -1.635684, -0.026360, -0.732545, ], + [-0.733555, 0.886134, -1.418841, -0.203192, -1.836270, ], + ... + [-1.172012, 1.678573, -0.004324, 0.441605, 1.484385, ], + [-0.550848, 1.455505, -0.968547, 1.233722, 0.812084, ]], + + [[ 0.762814, -0.641657, 0.843695, -1.380960, 0.261200, ], + [ 1.500137, -0.921287, 1.001725, 0.106540, -1.326231, ], + ... + [-0.878001, -0.465694, -1.204548, -0.407944, -0.092967, ], + [ 1.035407, 0.043012, -1.959517, 0.075913, 0.893392, ]], + + [[-0.129989, 0.750885, 0.219408, 0.442481, -0.508588, ], + [ 0.791268, -0.547978, 0.194177, 0.652729, 2.642380, ], + ... + [-0.674993, -1.017359, -1.272791, 0.829529, 0.852879, ], + [-0.256070, -0.287006, 1.900199, -0.492318, -0.658134, ]]]) + + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$src, + Optional:$coord, + DefaultValuedAttr:$verbose, + DefaultValuedAttr:$is_signed + ); + let results = (outs + ); +} +def Cute_PtrLoadOp : Cute_InferredOp<"ptr.load", [MemoryEffects<[MemRead]>]> { + let summary = "Loads an LLVM or CuTe value from the given `!cute.ptr`."; + let arguments = (ins + Cute_CuTePointerType:$ptr + ); + let results = (outs + Cute_CuTePointerTypeOrAMemoryReferenceType:$value + ); +} +def Cute_PtrStoreOp : Cute_Op<"ptr.store"> { + let description = [{ + Stores the given CuTe value at the provided pointer. Handles any + LLVM-storable type, including those that can be stored with + `llvm.store` *and* CuTe types. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrAMemoryReferenceType:$value, + Cute_CuTePointerType:$ptr + ); + let results = (outs + ); +} +def Cute_PtrToIntOp : Cute_Op<"ptrtoint", [Pure]> { + let description = [{ + The `cute.ptrtoint` operation accepts a cute pointer value and casts it to an + integer value. The pointer must have byte alignment, meaning subbyte types + must have explicit alignment specified (e.g., align<1> or higher). + + Example: + + ```mlir + %int_aligned = cute.ptrtoint(%ptr) : !cute.ptr> to i64 + ``` + }]; + let arguments = (ins + Cute_PointerTypeInterfaceInstance:$src + ); + let results = (outs + Cute_Integer:$dst + ); +} +def Cute_RakedProductOp : Cute_InferredOp<"raked_product", [Pure]> { + let description = [{ + This operation performs "raked_product" on the input and produce a raked_product result, + and it automatically infer the "raked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.raked_product(%arg0, %arg1) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((2,3),(5,4)):((12,4),(24,1))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.raked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((?,5),(?,4)):((?{div=20},4),(?{div=20},1))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_RandomDelayOp : Cute_Op<"random_delay"> { + let arguments = (ins + ); + let results = (outs + ); +} +def Cute_RecastIterOp : Cute_Op<"recast_iter", [Pure]> { + let description = [{ + This `cute.recast_iter` is similar to CuTe C++'s recast_ptr, convert two iterators + in certain aspects like type/swizzle for convenience. + But this operation has some usage limitation for correctness, + like it's not allowed for TMA tensor's iterator and it's not allowed to convert align attribute. + In general, it's not allowed to convert two different iterator kinds (PtrType <-> IntTupleType) + + Example: + + ```mlir + %ptr = cute.recast_iter(%0) : !cute.ptr, S<3,4,3>> + to !cute.ptr, S<2,4,3>> + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$src + ); + let results = (outs + Cute_CuTePointerType:$dst + ); +} +def Cute_RecastLayoutOp : Cute_InferredOp<"recast_layout", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + # upcast<8> operation + %0 = cute.recast_layout<32, 4>(%arg0) + : !cute.layout<"(128,64):(1,128)"> to !cute.layout<"(16,64):(1,16)"> + # upcast<2> for composed layout + %0 = cute.recast_layout<32, 16>(%arg0) + : !cute.composed_layout<"((6,7),8):((10,12),1) o 6 o (8,2):(1,4)"> + to !cute.composed_layout<"((6,7),4):((5,6),1) o 3 o (4,2):(1,2)"> + # downcast<8> operation + %0 = cute.recast_layout<8, 32>(%arg0) + : !!cute.layout<"(16,64):(1,16)"> to cute.layout<"(128,64):(1,128)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src, + I32Attr:$new_type_bits, + I32Attr:$old_type_bits + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$dst + ); +} +def Cute_ReduceOp : Cute_InferredOp<"reduce", [Pure]> { + let description = [{ + The `cute.reduce` operation reduces elements of a vector according to a given + shape and reduction operator, optionally guided by a reduction profile and/or a + predication mask. + + The reduction is performed over the dimensions of the input vector as described + by the provided shape. By default, all dimensions are reduced to a scalar. If a + reduction profile is specified, it determines which dimensions are kept (marked + with '_') and which are reduced (marked with an integer or tuple structure). The + result is either a scalar or a lower-rank vector, along with the reduced shape + if applicable. + + An optional predication vector (of the same size as the input vector) can be + provided to mask out elements from the reduction, enabling conditional + reduction. + + The reduction operator (e.g., MIN, MAX, ADD, MUL) is specified via the `op` + attribute. + + Examples: + ```mlir + // Reduce all elements to a scalar (minimum) + %reduced = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32) -> f32 + + // Reduce all elements to a scalar (maximum) with predication + %reduced = cute.reduce(%input, %shape, %init, %pred) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32, vector<20xi1>) -> f32 + + // Reduce all but the second dimension (profile keeps dimension 1) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(1,_,1)"> } + : (vector<60xf32>, !cute.shape<"(5,4,3)">, f32) -> (vector<4xf32>, !cute.shape<"(4)>") + + // Reduce a nested dimension (profile keeps outer and first inner dimension) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(_,(_,1))"> } + : (vector<30xf32>, !cute.shape<"(5,(3,2))">, f32) -> (vector<15xf32>, !cute.shape<"(5,3)>") + ``` + The operation supports both flat and nested shapes and profiles, enabling reductions over complex multidimensional data layouts. + }]; + let arguments = (ins + Cute_OfRanks1:$input, + AnyType:$shape, + AnyType:$init_val, + Optional:$predication, + Cute_ReductionOpAttr:$reduction_op, + OptionalAttr:$reduction_profile + ); + let results = (outs + Cute_OfRanks1OrFloatingPointOrInteger:$output, + Optional:$reduced_shape + ); +} +def Cute_RightInverseOp : Cute_InferredOp<"right_inverse", [Pure]> { + let description = [{ + This operation builds the right inverse of a layout. + The current implementation only supports static layout. + An error is returned when the layout contains dynamic values. + + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(8, 4):(4, 1)"> + %inv_layout = cute.right_inverse %layout : (!cute.layout<"(8, 4):(4, 1)">) -> !cute.layout<"(4, 8):(8, 1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SelectOp : Cute_InferredOp<"select", [Pure]> { + let description = [{ + This operation extracts specific dimensions out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a coordinate, a tuple, a layout, ... + + The `mode` attribute indices are used to select the portion of the input type. + The indices in the `mode` attribute can be permutated or repeated, but should + be less than the rank of type. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.select<[1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"((2, 4))"> + %mode1 = cute.select<[0,1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(1, (2,4))"> + %mode2 = cute.select<[2,0]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 1)"> + %mode3 = cute.select<[2,2]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 8)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + DenseI32ArrayAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ShapeDivOp : Cute_InferredOp<"shape_div", [Pure]> { + let description = [{ + Division for Shapes. + Case Tuple Tuple: + Perform shape_div element-wise + Case Tuple Int: + Fold the division of b across each element of a + Example: shape_div((4,5,6),40) -> shape_div((1,5,6),10) -> shape_div((1,1,6),2) -> (1,1,3) + Case Int Tuple: + Return shape_div(a, product(b)) + Case Int Int: + Enforce the divisibility condition a % b == 0 || b % a == 0 when possible + Return a / b with rounding away from 0 (that is, 1 or -1 when a < b) + + ```mlir + %res = cute.shape_div(%a, %b) + : (!cute.int_tuple<"(4,5)">, !cute.int_tuple<"(40)">) -> !cute.int_tuple<"(1,1)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$a, + Cute_CuteShapeType:$b + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_SizeOp : Cute_InferredOp<"size", [Pure]> { + let summary = "This operation computes the size of a cute value type by multiplying its dimensions."; + let description = [{ + The `mode` attribute specifies the nested mode in the type for size calculation. + It indexes a sub-scope within the input type and returns the size of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the size of the entire type. + + ### Examples + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),9)"> + %size = cute.size(%shape) <{mode=[1]}> : (!cute.shape<"(1,(2,4),9)">) -> !cute.int_tuple<"8"> + ``` + + In this example, the mode attribute is set to [1], which selects the second dimension of the + shape (1,(2,4),9). The size of this dimension is calculated as 2 * 4 = 8. + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),8)"> + %size = cute.size(%shape) : (!cute.shape<"(1,(2,4),8)">) -> !cute.int_tuple<"64"> + ``` + + In this example, the mode attribute is empty, so the operation calculates the size of the entire + shape (1,(2,4),8). The size is calculated as 1 * 2 * 4 * 8 = 64. + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr2:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_SliceOp : Cute_InferredOp<"slice", [Pure]> { + let description = [{ + This operation takes slices of a input memref according to the `coord`. Note that + `!coord<"(_,_,k)">` means "create a slice that views (i, j, k) for all valid i, + all valid j, and a specific value of k. + + The `coord` elements can be static, dynamic (represented by "?"), or underscore + (similar to the ":" (the colon punctuation) in Python or Fortran array slices). + However, the mode of `coord` should be less than or equal to the mode of `input` + memref layout; otherwise, a runtime error will occur. + + Example: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(0, (_, _), 0)"> + ``` + + Example with dynamic coord: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(_,(64,?),0)"> + ``` + + Example with scaled basis in layout: + + ```mlir + %slice = cute.slice(%coord_tensor, %coord) + : !cute.coord_tensor<"(0,0)", "(10,20,30):(1@0,1@1,20@1)">, !cute.coord<"(0,2,_)"> + ``` + Example with swizzled layout: + + ```mlir + %slice = cute.slice(%swizzle_layout, %coord) + : (!cute.composed_layout<"S<1,5,1> o 0 o ((2,4),(8,4)):((1,16),(2,64))">, + !cute.coord<"((?,?),(_,_))">) -> !cute.composed_layout<"(8,(2,2)):(2,(?,128))"> + ``` + + Example with PDSL memref slice (swizzle in units of bytes): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref, "(64,32):(32,1)">, !cute.coord + ``` + + Example with PISL memref slice (swizzle in units of elements): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref o 0 o (64,32):(32,1)">, !cute.coord + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SmemPartitionSizeOp : Cute_Op<"smem_partition_size", [Pure]> { + let description = [{ + This operation queries the static smem layout for one partition in the + current kernel and returns the partition size as an i64 integer. It is only + valid on the kernel side; host-side total smem queries should use + `cute.kernel_smem_size`. + }]; + let arguments = (ins + I32Attr:$partition_id, + UnitAttr:$cumulative + ); + let results = (outs + I64:$size + ); +} +def Cute_StaticOp : Cute_Op<"static", [Pure]> { + let description = [{ + Creates a compile-time constant value with a fully static CuTe type. This operation takes no + operands and produces a single result of one of the following CuTe types: + + - !cute.int_tuple: A tuple of static integers + - !cute.coord: A coordinate with static dimensions + - !cute.stride: A stride with static values + - !cute.shape: A shape with static dimensions + - !cute.tile: A tile with static dimensions and layout + - !cute.layout: A layout with static shape and strides + - !cute.composed_layout: A composed layout with static inner/outer layouts + + The type of the result must be fully static (contain no dynamic/unknown values). + + Some atom types can be fully static with no runtime states, e.g. `UniversalFma`. + + ### Examples: + + ```mlir + // Create a static shape with value 5 + %0 = cute.static : !cute.shape<"5"> + + // Create a static 2D layout with shape (32,64) and strides (64,1) + %1 = cute.static : !cute.layout<"(32,64):(64,1)"> + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MaybeStaticTypeInterfaceInstance:$result + ); +} +def Cute_StencilDivideOp : Cute_InferredOp<"stencil_divide", [AttrSizedOperandSegments, Pure]> { + let summary = "A convolution has the following input tensors:"; + let description = [{ + * Activation tensor (Act): `((N,(D,H,W,...)), (C,(1,1,1)))` + * Filter tensor (Flt): `(K,(C,(T,R,S, ...)))` + * Output tensor (Out): `((N,(Z,P,Q, ...)), K)` + + Where + + * `N` := number of samples in the signal + * `D`, `H`, `W`, ... := spatial dimensions of the activation tensor + * `C` := channel dimension of the activation tensor + * `K` := channel dimension of the filter and output tensor + * `T`,`R`,`S`,... := spoke dimensions of the filter tensor + * `Z`,`P`,`Q`,... := spatial dimensions of the output tensor + + The number of spatial dimensions must DHW... must match both TRS... and ZPQ... + + As is evident in the tensor shapes, these cannot be issued to a GEMM without transformation, since + there are no logical M, N, and K modes into which we can group the tensor modes. + + Every spoke of the filter tensor (TRS) will be applied to some (offset) + view of the activation tensor, thus expanding the logical size of the activation tensor. + Additionally, a similar logical transform of the spatial dimensions can be encoded as + a function of the padding, dilations, traversal strides, and filter spokes. + This gets us to our im2col transform: + + Im2col transform affects the component shapes/strides of the activation tensor in the following way: + + * `ZPQ` Shape : changes `DHW` domain with formula `(1 + (DHW + pad - (((TRS-1) * dilation) + 1)) / traversal_stride)` + * `TRS` Shape : `TRS` domain instead of `(1,1,1)` + * `ZPQ` Strides : Original `DHW` strides get scaled by traversal strides `DHW` + * `TRS` Strides : Original `DHW` strides get scaled by dilation `DHW`. + + With this transform applied, we end up with a set of input and output tensors that are logically + consistent in their `MNK` dimensions, thus allowing us to dispatch to a GEMM. + + * im2col activation layout: `((N,(Z,P,Q)), (C,(T,R,S)))` // logical (M,K) + * filter layout : `( K, (C,(T,R,S)))` // logical (N,K) + * output layout : `((N,(Z,P,Q)), K )` // logical (M,N) + + `stencil_divide` performs this operation given these parameters. + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteTileType:$stencil, + Optional:$padding_upper, + Optional:$padding_lower, + Optional:$traversal_stride + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SymbolicOp : Cute_Op<"symbolic", [Pure]> { + let description = [{ + A symbol in CuTe IR represent an abstracted value which could be materialized + later in a pass. If it's not materialized, the behavior is undefined. + + Example: + + ```mlir + %0 = cute.symbolic("tile_m") : i32 + %1 = cute.symbolic("cluster_size") : !cute.shape<"(?,2)"> + ``` + }]; + let arguments = (ins + StrAttr:$name + ); + let results = (outs + AnyType:$symbolic + ); +} +def Cute_TileToShapeOp : Cute_InferredOp<"tile_to_shape", [Pure]> { + let description = [{ + The `tile_to_shape` operation is similar to blocked_product, but specifies the + result shape instead of the product shape, which is more convenient in certain + circumstances. + + The op has three operands: + 1. `block`: The layout to repeat. + 2. `trg_shape`: The target shape of the result + 3. `ord_shape`: The order of the modes of `trg_shape` to tile layout `block` with. + E.g. (_2,_1,_3) will cause @a layout to repeat across the second mode first, + the first mode second, and the third mode last. Currently only supports static order. + + Example: + + ```mlir + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"(128,128):(128,1)">, + !cute.shape<"(1024,512)">, + !cute.shape<"(2,1)">) + -> !cute.layout<"((128,8),(128,4)):((128,65536),(1,16384))"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(1,2,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,512),(0,1,1024),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(2,1,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,4096),(0,1,512),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.composed_layout<"S<3,4,3> o 0 o ((32,4),(?,4)):((16,4),(0,1))">, + !cute.shape<"(256,?,2)">, + !cute.shape<"(3,2,1)">) + -> !cute.composed_layout<"S<3,4,3> o 0 o ((32,4,2),(?,4,?),2):((16,4,?{div=1024}),(0,1,1024),512)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$block, + Cute_CuteShapeType:$trg_shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$ord_shape + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledCopyPartitionDOp : Cute_InferredOp<"tiled.copy.partition_D", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyPartitionSOp : Cute_InferredOp<"tiled.copy.partition_S", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyRetileOp : Cute_InferredOp<"tiled.copy.retile", [Pure]> { + let description = [{ + The cute.tiled.copy.retile operation is used to reorganize the layout of a tensor + that has already been tiled or partitioned. This is typically necessary when + two TiledMma or TiledCopy instances are connected, accessing the same memory, + but require different logical views of that memory. + + ### Use Case + + Consider a scenario where a TiledMma instance is the consumer, and a TiledCopy + instance is the producer. The TiledMma instance partitions the memory, creating + a logical view that suits its needs. However, this view may not be compatible with + the TiledCopy instance. To resolve this, the cute.tiled.copy.retile operation is used + to retile the TiledCopy instance, creating a new logical view that is compatible with + the TiledMma instance. + + ``` + // consumer + partition_A(%tiled_mma, ...) + %mma_view = partition_A(%tiled_mma, %sa_tensor, %idx) + + // producer + %tiled_copy = make_tiled_copy_A(%copy_atom, %tiled_mma) + // retile %mma_view to meet needs of tiled_copy + %lds_view = retile(%tiled_copy, %mma_view) + ``` + + ### Operation + + The cute.tiled.copy.retile operation takes two inputs: + * A cute.tiled_copy instance, which represents the tensor to be retiled. + * A CuTeTypeWithLayout derived type, which represents the tensor that has already been tiled or partitioned. + + + Example: + + ```mlir + %retiled = cute.tiled.copy.retile(%tiled_copy, %src_tensor) + : (!cute.tiled_copy, num_matrices=4>, + layout_copy_tv = <"((4,8,4),((2,2,4),1)):((128,4,1),((64,32,512),0))">, + tiler_mn = <"[(4,16):(32,1);32:1]">>, + !cute.memref) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledDivideOp : Cute_InferredOp<"tiled_divide", [Pure]> { + let summary = "This operation performs a tiled_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledMmaPartitionOp : Cute_InferredOp<"tiled.mma.partition", [Pure]> { + let description = [{ + `cute.tiled.mma.partition` reorganizes a tensor's layout to match what the + MMA atom expects. The operation takes 4 arguments: + + 1. The type of operand. E.g., `CUTE_MMA_OPERAND_A` or `CUTE_MMA_OPERAND_B`. + 2. A `!cute.tiled_mma` value that specifies the kind of MMA. + 3. The unpartitioned operand tensor (ViewTypeInterface-derived SSA value). + 4. The `!cute.coord` value of the execution unit for which to partition the + the input tensor. + + The coordinate type depends on the MMA atom and the operand memory space. + Some examples: + * SM80: thread ID + * SM90: It depends... + - When partitioning A with "frag_kind = ss": warp-group ID + - When partitioning A with "frag_kind = rs": thread ID + - When partitioning B: warp-group ID + - When partitioning C/D: thread ID + * SM100: CTA ID within the cluster (always 0 for 1-CTA mode) + + The operation returns a new ViewTypeInterface-derived value + (CuteMemrefType/CuteCoordTensorType) with partitioned layout and sliced + offset (sliced offset will be implemented in the lowering). + + Example: + + ```mlir + // Partition an operand A tensor of shape 128x32 (residing in shared + // memory) for a !tiled_mma atom that operates on shape 32x16x16. (I.e., + // four SM80 MMA atoms with shape 16x8x16.) The result tensor is a view + // into the A tensor that represents the part that thread %tid_coord is + // operating on. + // + // The last two modes of the result tensor are 4,2. This is because the + // tiled atom is applied 4 times in M direction and 2 times in K direction. + // + // K=0:15 K=16:31 + // ┌────────────────────┬────────────────────┐ + // M=0:31 │ tiled atom (0,0) │ tiled atom (0,1) │ + // ├────────────────────┼────────────────────┤ + // M=32:63 │ tiled atom (1,0) │ tiled atom (1,1) │ + // ├────────────────────┼────────────────────┤ + // M=64:95 │ tiled atom (2,0) │ tiled atom (2,1) │ + // ├────────────────────┼────────────────────┤ + // M=96:127 │ tiled atom (3,0) │ tiled atom (3,1) │ + // └────────────────────┴────────────────────┘ + // + // For each tiled atom application of size 32x16x16, thread %tid_coord gets + // a subview of size 2,2,2 of operand A. (128 threads are cooperatively + // computing the !tiled_mma of shape 32x16x16, but an individual thread + // does not need access to the full 32x16 matrix A part.) + // + %partitioned = cute.tiled.mma.partition A(%0, %1, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // Partition an operand A tensor of shape 256x512 for an SM100 !tiled_mma. + // There are 256/128=2 tiled atom applications in M direction and + // 512/16=32 tiled atom applications in K direction. The atom specifies a + // single CTA, so that CTA get the entire A tensor. The coordinate is not + // really need in this case, but we pass it anyway to keep the API uniform. + %partitioned = cute.tiled.mma.partition A (%0, %1, %cta_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "(256,512):(512,1)">, + !cute.coord<"0">) + -> !cute.memref, "((128,16),2,32):((512,1),65536,16)"> + ``` + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledMmaPartitionShapeOp : Cute_InferredOp<"tiled.mma.partition_shape", [Pure]> { + let description = [{ + This operation partitions a shape into an MMA shape. For example, it takes a logical (M,K,...) + shape and tiles it into a shape like (MMA,MMA_M,MMA_K,...) where + - MMA is the MMA mode, for example (128,16) for an MMA instruction of shape 128x128x16 + - MMA_M is the number of times the MMA repeats in the M-mode + - MMA_K is the number of times the MMA repeats in the K-mode + - ... denotes the "rest" + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_CuteShapeType:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_TiledProductOp : Cute_InferredOp<"tiled_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ToIntTupleOp : Cute_InferredOp<"to_int_tuple", [Pure]> { + let description = [{ + The `cute.cast_to_int_tuple` operation casts from any x-tuple type (shape, stride, coord) to an int-tuple type. + The operation preserves the values but changes their type to int-tuple. + + cast will fail if input types contains modes not supported by int-tuple, e.g. _, scaled based, etc. + + Example: + + ```mlir + // Cast from shape to int_tuple + %0 = cute.to_int_tuple(%shape) : !cute.shape<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple + %1 = cute.to_int_tuple(%stride) : !cute.stride<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from coord to int_tuple + %2 = cute.to_int_tuple(%coord) : !cute.coord<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple preserving divisibility + %3 = cute.to_int_tuple(%stride) : !cute.stride<"(8,?{div=4})"> to !cute.int_tuple<"(8,?{div=4})"> + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$src + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$dst + ); +} +def Cute_TupleAddOp : Cute_InferredOp<"tuple_add", [Pure]> { + let description = [{ + The `cute.tuple_add` operation performs element-wise addition between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the sum of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise add two arithmetic tuples + %result = cute.tuple_add(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Element-wise add with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_add(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleDivOp : Cute_InferredOp<"tuple_div", [Pure]> { + let description = [{ + The `cute.tuple_div` operation performs element-wise division between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the quotient of the corresponding elements in the input tuples. + + This operation applies division recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise divide two arithmetic tuples + %result = cute.tuple_div(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise divide with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_div(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(0,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleModOp : Cute_InferredOp<"tuple_mod", [Pure]> { + let description = [{ + The `cute.tuple_mod` operation computes the element-wise modulo (remainder) + between two CuTe x-tuple types. Both operands must be of the same tuple kind + (e.g., `!cute.int_tuple`), and the result is a tuple of the same structure + and type. + + For each corresponding element in the input tuples, the operation computes + the remainder of the division of the left-hand side element by the right-hand + side element. If the tuples are nested, the operation is applied recursively, + matching elements at each level of the tuple hierarchy. + + This operation supports both static and dynamic tuple elements. When static + values are provided, the modulo is computed at compile time; otherwise, it is + performed at runtime. + + **Examples:** + + ```mlir + // Element-wise modulo of two arithmetic tuples + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) + -> !cute.int_tuple<"(?,(0,?))"> + + // Element-wise modulo with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(1,?))"> + ``` + + **Semantics:** + - The operation traverses the tuple structure recursively. + - For each pair of corresponding elements `(x, y)`, the result is `x % y`. + - The structure and rank of the result tuple matches that of the operands. + - If the tuple contains nested tuples, the modulo is applied to each + corresponding sub-tuple. + + This operation is useful for broadcasting, tiling, and other tensor algebra + applications where modular arithmetic over structured tuples is required. + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleMulOp : Cute_InferredOp<"tuple_mul", [Pure]> { + let description = [{ + The `cute.tuple_mul` operation performs element-wise multiplication between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the product of the corresponding elements in the input tuples. + + This operation applies multiplication recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise multiply two arithmetic tuples + %result = cute.tuple_mul(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise multiply with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mul(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(3,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleProductEachOp : Cute_InferredOp<"tuple.product_each", [Pure]> { + let summary = "This operation performs \"product_each\" on each element in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.tuple.product_each(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"(2, 20)"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleProductOp : Cute_InferredOp<"tuple.product", [Pure]> { + let summary = "This operation performs \"product\" on all elements in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.product(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"40"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleSubOp : Cute_InferredOp<"tuple_sub", [Pure]> { + let description = [{ + The `cute.tuple_sub` operation performs element-wise subtraction between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the difference of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise subtract two arithmetic tuples + %result = cute.tuple_sub(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(2,?))"> + + // Element-wise subtract with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_sub(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(2,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_UnpackTupleOp : Cute_Op<"unpack_tuple", [Pure]> { + let description = [{ + Unpacks a tuple into its individual components. The number and types + of the components must match the number and types of the results. + }]; + let arguments = (ins + Cute_ATupleIsAFixedLengthOrderedList:$tuple + ); + let results = (outs + Variadic:$result + ); +} +def Cute_ZippedDivideOp : Cute_InferredOp<"zipped_divide", [Pure]> { + let description = [{ + This operation performs "zipped_divide" on the input and produce a zipped_divided result, + and it automatically infer the "zipped_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.zipped_divide(%alloc, %shape) + : !cute.memref, !cute.shape<"(4,8)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ZippedProductOp : Cute_InferredOp<"zipped_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD diff --git a/mlir/dialects/cute_ir/CuteTypes.cpp b/mlir/dialects/cute_ir/CuteTypes.cpp new file mode 100644 index 00000000..604dfbf2 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypes.cpp @@ -0,0 +1,1602 @@ +//===- CuteTypes.cpp - parsers, printers and verifiers of cute types ------===// + +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; + +namespace cg = cutegen; + +// The compiler's parser of the algebra types reads `<"text">` leniently: a +// missing or unreadable payload, including a bare mnemonic, gives the default +// value (`()`, `():()`, `[]`, `S<0,4,3>`) instead of an error. +template static T parseLenientPayload(mlir::AsmParser &parser) { + std::string str; + if (succeeded(parser.parseOptionalLess()) && + succeeded(parser.parseOptionalString(&str)) && + succeeded(parser.parseOptionalGreater())) + return cutegen::from_string(str).value_or(T{}); + return T{}; +} + +// The compiler's integer type is `!cute.i`, a mnemonic per width that +// the generated dispatch cannot match. The generated CuteDialect::parseType +// and printType pass the dialect parser and printer, so these overloads of +// the renamed dispatch functions catch them first (defined below). +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value); +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer); +#define generatedTypeParser generatedCuteTypeParser +#define generatedTypePrinter generatedCuteTypePrinter + +// Emit the tablegen-generated type storage / parsing machinery (also drives +// CuteDialect::parseType / printType via useDefaultTypePrinterParser). +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" +#undef generatedTypeParser +#undef generatedTypePrinter + +//===----------------------------------------------------------------------===// +// CuteDialect — type registration +//===----------------------------------------------------------------------===// + +namespace { +namespace cute = ::mlir::cutlass_compiler::cute; +namespace nv = ::mlir::cutlass_compiler::cute_nvgpu; + +std::string printed(Type type) { + std::string text; + llvm::raw_string_ostream os(text); + type.print(os); + return text; +} + +// The compiler's name for a tiled copy's atom; empty when it has none. +std::string copyAliasName(Type atom) { + return llvm::TypeSwitch(atom) + .Case([](auto) { return "copy_simt"; }) + .Case([](auto) { return "copy_ldgsts"; }) + .Case([](auto) { return "copy_dsmem_store"; }) + .Case([](auto) { return "copy_blkcp_g2s"; }) + .Case([](auto) { return "copy_blkcp_s2g"; }) + .Case([](auto) { return "copy_blkcp_s2s"; }) + .Case([](auto) { return "copy_g2r"; }) + .Case([](auto) { return "copy_r2g"; }) + .Case([](auto) { return "copy_r2s"; }) + .Case([](auto) { return "copy_s2r"; }) + .Case([](auto) { return "copy_utccp"; }) + .Case( + [](auto t) { return "copy_ldtm_" + std::to_string(t.getNumBit()); }) + .Case( + [](auto t) { return "copy_sttm_" + std::to_string(t.getNumBit()); }) + .Case([](auto t) { + return "copy_ldtm_red_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldtm_spcompress_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_stsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_tmaldg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmastg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmaredg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Default([](Type) { return "copy_unknown_type"; }); +} + +// `mma____xx`; empty for a type that is not an MMA atom. +std::string mmaAliasName(Type atom) { + auto name = [](auto t) { + std::string shape; + auto mnk = cutegen::flatten(t.getShapeMnk().getRef()); + for (size_t i = 0; i < cutegen::rank(mnk); ++i) + shape += (i ? "x" : "") + cutegen::to_string(cutegen::get(mnk, i)); + return "mma_" + printed(t.getAType()) + "_" + printed(t.getBType()) + "_" + + printed(t.getCType()) + "_" + shape; + }; + return llvm::TypeSwitch(atom) + .Case(name) + .Default([](Type) { return std::string(); }); +} + +// The compiler's type aliases: `!memref__` for a memref, +// `!copy_` for a tiled copy and `!mma____xx` for a +// tiled MMA (MLIR escapes and numbers them). Other types have none. +struct CuteOpAsmInterface : public OpAsmDialectInterface { + using OpAsmDialectInterface::OpAsmDialectInterface; + AliasResult getAlias(Type type, raw_ostream &os) const override { + std::string name; + if (auto memref = llvm::dyn_cast(type)) + name = "memref_" + + stringifyAddressSpace(memref.getPtr().getAddressSpace()).str() + + "_" + printed(memref.getPtr().getValueType()); + else if (auto copy = llvm::dyn_cast(type)) + name = copyAliasName(copy.getCopyAtom()); + else if (auto mma = llvm::dyn_cast(type)) + name = mmaAliasName(mma.getMmaAtom()); + if (name.empty()) + return AliasResult::NoAlias; + os << name; + return AliasResult::OverridableAlias; + } +}; +} // namespace + +void CuteDialect::registerCuteTypes(CuteDialect *dialect) { + dialect->addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" + >(); + dialect->addInterfaces(); +} + +//===----------------------------------------------------------------------===// +// IntTupleType — custom assembly format +// + +// Grammar: < " cutegen-int-tuple-string " > (read leniently, see above) +//===----------------------------------------------------------------------===// + +mlir::Type IntTupleType::parse(mlir::AsmParser &parser) { + return get( + parser.getContext(), + IntTupleAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void IntTupleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool IntTupleType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute IntTupleType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +IntTupleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +IntTupleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +IntTupleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool IntTupleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + // The lowering can be a bare integer (scalar/depth-0 dynamic leaves), which + // does not implement DataLayoutTypeInterface; only delegate when it does and + // otherwise fall back to the interface default ("compatible"). + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +IntTupleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + // See areCompatible: bare-integer lowerings cannot be queried, so only + // delegate when the lowered type implements the interface. + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// CoordType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type CoordType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + CoordAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void CoordType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool CoordType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute CoordType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +CoordType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +CoordType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t CoordType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +CoordType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +CoordType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool CoordType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +CoordType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ShapeType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ShapeType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + ShapeAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void ShapeType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ShapeType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute ShapeType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +ShapeType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +ShapeType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ShapeType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ShapeType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +ShapeType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ShapeType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ShapeType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// StrideType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type StrideType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = StrideAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void StrideType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool StrideType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute StrideType::getValueAttr() const { return getAttr(); } + +// The compiler accepts every stride cutegen reads, mixed integer and +// scaled-basis leaves and mixed-depth basis paths included. +mlir::LogicalResult +StrideType::verify(llvm::function_ref emitError, + StrideAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +StrideType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +StrideType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +StrideType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool StrideType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +StrideType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// LayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type LayoutType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = LayoutAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void LayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool LayoutType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute LayoutType::getValueAttr() const { return getAttr(); } + +/// Verify LayoutType storage invariants. cutegen leaves these +/// unchecked at construction time for performance; enforce them at the +/// IR boundary. +mlir::LogicalResult +LayoutType::verify(llvm::function_ref emitError, + LayoutAttr attr) { + auto const &layout = attr.getRef(); + if (cutegen::has_error(layout)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (cutegen::any_leaf_is(layout.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + })) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (!cutegen::is_congruent(layout.shape(), layout.stride())) { + return emitError() + << "expects shape and stride profile to match, but got \"" + << cutegen::to_string(layout) << "\""; + } + return mlir::success(); +} + +llvm::TypeSize +LayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +LayoutType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +LayoutType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool LayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +LayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// TileType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type TileType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = TileAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void TileType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool TileType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute TileType::getValueAttr() const { return getAttr(); } + +// Like strides, the layouts of a tile are not restricted further. +mlir::LogicalResult +TileType::verify(llvm::function_ref emitError, + TileAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +TileType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +TileType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t TileType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +TileType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +TileType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool TileType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +TileType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ComposedLayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ComposedLayoutType::parse(mlir::AsmParser &parser) { + // Capture the start location so verify() failures (positivity, + // congruence) are attributed to the type literal. + auto loc = parser.getCurrentLocation(); + if (parser.parseLess()) { + return {}; + } + std::string str; + if (parser.parseString(&str)) { + return {}; + } + auto opt = cutegen::from_string(str); + if (!opt) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse composed_layout from \"" + str + "\""); + return {}; + } + if (parser.parseGreater()) { + return {}; + } + auto attr = ComposedLayoutAttr::get(parser.getContext(), std::move(*opt)); + return ComposedLayoutType::getChecked([&] { return parser.emitError(loc); }, + parser.getContext(), attr); +} + +void ComposedLayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ComposedLayoutType::isStatic() const { + // No free cutegen::is_static(composed_layout_t) exists — check each part. + const auto &cl = getRef(); + bool b_ok = cutegen::is_static(cl.layout_b()); + bool off_ok = cutegen::is_static(cl.offset()); + bool a_ok = cl.is_a_swizzle() || cutegen::is_static(cl.layout_a()); + return b_ok && off_ok && a_ok; +} + +mlir::Attribute ComposedLayoutType::getValueAttr() const { return getAttr(); } + +/// Verify ComposedLayoutType storage invariants: no error sentinel, +/// positive static shape leaves, addable offset, and a sum that's +/// composable into the inner layout (scalar for swizzle, scalar or +/// weakly congruent for affine). +mlir::LogicalResult ComposedLayoutType::verify( + llvm::function_ref emitError, + ComposedLayoutAttr attr) { + auto const &composed = attr.getRef(); + if (cutegen::has_error(composed)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(composed) << "\""; + } + // A trivial wrap (identity swizzle, no offset) is accepted and prints as + // its plain layout, as the compiler does. + auto hasNonPositiveShape = [](cutegen::layout const &lyt) { + return cutegen::any_leaf_is(lyt.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + }); + }; + if (hasNonPositiveShape(composed.layout_b()) || + (composed.is_a_affine() && hasNonPositiveShape(composed.layout_a()))) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(composed) << "\""; + } + + // Addable: the offset and layout_eval(0, outer) must be compatible + // under arith-tuple addition (no scaled-basis-vs-integer mismatch + // or rank disagreement). + auto offset_b = cutegen::layout_eval(0, composed.layout_b()); + auto sum = + cutegen::arith_tuple_sum(composed.offset(), offset_b); + if (cutegen::has_error(sum)) { + return emitError() << "expects offset and `layout_eval(0, outer)` to be " + "addable, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(offset_b); + } + + // Swizzle inner: the arith-tuple sum must be a scalar (int or + // dynamic int) — swizzles can only consume scalar indices, so any + // scaled-basis structure surviving from the offset or outer is a + // composability error. + if (composed.is_a_swizzle() && !cutegen::holds_int_or_dynamic_int(sum)) { + return emitError() << "swizzle layout expects scalar `offset` and " + "`outer` without scaled basis, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(composed.layout_b()); + } + + // Affine inner: the sum either is a scalar (and layout_a indexes + // it directly) or matches layout_a's shape profile so each leaf + // dispatches to the corresponding mode of A. + if (composed.is_a_affine() && !cutegen::holds_int_or_dynamic_int(sum) && + !cutegen::weakly_congruent(sum, composed.layout_a().shape())) { + return emitError() + << "expects arith sum of `layout_eval(0, outer)` and `offset` to be " + "scalar or weakly congruent to inner shape, but got sum " + << cutegen::to_string(sum) << " and inner shape " + << cutegen::to_string(composed.layout_a().shape()); + } + + return mlir::success(); +} + +llvm::TypeSize +ComposedLayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize ComposedLayoutType::getTypeSizeInBits( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ComposedLayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ComposedLayoutType::getPreferredAlignment( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional ComposedLayoutType::getIndexBitwidth( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ComposedLayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ComposedLayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// SwizzleType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type SwizzleType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = SwizzleAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void SwizzleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool SwizzleType::isStatic() const { return true; } + +mlir::Attribute SwizzleType::getValueAttr() const { return getAttr(); } + +/// Verify SwizzleType storage invariants. Defense-in-depth — the +/// string parser already rejects invalid swizzle parameters via +/// cutegen::from_string, but programmatic +/// construction goes around the parser, so re-check here. +/// +/// cutegen::swizzle::is_valid (swizzle.hpp) enforces two +/// invariants on S: +/// 1. |num_shift| + num_bits + num_base <= 32 — parameters +/// must fit within the swizzle's 32-bit working width. +/// 2. |num_shift| >= num_bits — the shift amount must cover +/// the mask width so the Y and Z masks do not overlap. +mlir::LogicalResult +SwizzleType::verify(llvm::function_ref emitError, + SwizzleAttr attr) { + auto const &sw = attr.getRef(); + uint32_t numBits = sw.num_bits(); + uint32_t numBase = sw.num_base(); + int32_t numShift = sw.num_shift(); + uint32_t absShift = static_cast(std::abs(numShift)); + constexpr uint32_t kBitWidth = 32; + + if (absShift + numBits + numBase > kBitWidth) { + return emitError() << "expects |num_shift| + num_bits + num_base <= " + << kBitWidth << ", but got " << absShift << " + " + << numBits << " + " << numBase << " = " + << (absShift + numBits + numBase) << " for swizzle " + << cutegen::to_string(sw); + } + if (absShift < numBits) { + return emitError() << "expects |num_shift| >= num_bits, but got " + << absShift << " < " << numBits << " for swizzle " + << cutegen::to_string(sw); + } + return mlir::success(); +} + +// A swizzle is a compile-time-only object whose runtime representation is an +// empty LLVM struct (mirroring convertSwizzleType in CuteTypeConverter.cpp), so +// the data-layout queries delegate to that struct rather than to +// cg::get_llvm_type. +static mlir::Type getSwizzleLLVMType(mlir::MLIRContext *ctx) { + return LLVM::LLVMStructType::getLiteral(ctx, SmallVector{}); +} + +llvm::TypeSize +SwizzleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSize(getSwizzleLLVMType(getContext())); +} + +llvm::TypeSize +SwizzleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSizeInBits(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeABIAlignment(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypePreferredAlignment(getSwizzleLLVMType(getContext())); +} + +std::optional +SwizzleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeIndexBitwidth(getSwizzleLLVMType(getContext())); +} + +bool SwizzleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +SwizzleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// The DSL compiler's types and attributes (CuteTypesCompiler.td, +// CuteAttrsCompiler.td) +//===----------------------------------------------------------------------===// + +#include "mlir/dialects/cute_ir/CuteEnums.cpp.inc" + +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value) { + OptionalParseResult result = generatedCuteTypeParser( + static_cast(parser), mnemonic, value); + if (result.has_value()) + return result; + StringRef digits = *mnemonic; + unsigned width; + if (!digits.consume_front("i") || digits.empty() || + !llvm::all_of(digits, llvm::isDigit) || digits.getAsInteger(10, width)) + return std::nullopt; + value = ConstrainedIntType::parse(parser, width); + return success(!!value); +} + +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer) { + if (auto integer = llvm::dyn_cast(type)) { + integer.print(printer); + return success(); + } + return generatedCuteTypePrinter(type, static_cast(printer)); +} + +// Bits of an integer or float (tf32 is stored in 32), or of a vector of them. +static unsigned storageBits(Type type) { + if (auto integer = llvm::dyn_cast_or_null(type)) + return integer.getWidth(); + if (auto fp = llvm::dyn_cast_or_null(type)) + return fp.isTF32() ? 32 : fp.getWidth(); + if (auto vector = llvm::dyn_cast_or_null(type)) + return vector.getNumElements() * storageBits(vector.getElementType()); + return 0; +} + +static bool isIdentitySwizzle(SwizzleAttr swizzle) { + return !swizzle || cutegen::to_string(swizzle.getRef()) == "S<0,4,3>"; +} + +uint64_t PtrType::getNaturalAlignment(Type valueType) { + unsigned bits = 0; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) + bits = storageBits(sparse.getPhysicalType()); + else if (llvm::isa_and_nonnull(valueType)) + bits = storageBits(valueType); + if (bits <= 1) + return 1; + return bits % 8 ? 0 : bits / 8; +} + +BitLayoutAttr PtrType::getDefaultBitLayout(Type valueType, + AddressSpace addressSpace) { + unsigned chunk = addressSpace == AddressSpace::tmem ? 32 : 8; + std::string text; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) { + unsigned bits = storageBits(sparse.getPhysicalType()); + if (!bits) + return {}; + text = llvm::formatv("({0},({1},{2})):(1,(0,{0}))", bits, + sparse.getNumLogical(), chunk / std::gcd(bits, chunk)) + .str(); + } else if (llvm::isa_and_nonnull(valueType)) { + unsigned bits = storageBits(valueType); + unsigned stride = bits == 1 ? 8 : bits; + text = llvm::formatv("({0},{1}):(1,{2})", bits, + chunk / std::gcd(stride, chunk), stride) + .str(); + } else { + return {}; + } + auto layout = cutegen::from_string(text); + if (!layout) + return {}; + MLIRContext *ctx = valueType.getContext(); + return BitLayoutAttr::get(ctx, LayoutAttr::get(ctx, std::move(*layout))); +} + +// The bits of a chunk: the largest extent times stride of the element mode. +static std::optional chunkBits(const cutegen::layout &layout) { + auto elements = cutegen::get(layout, 1); + auto shape = cutegen::flatten(elements.shape()); + auto stride = cutegen::flatten(elements.stride()); + if (cutegen::holds_int(shape) && cutegen::holds_int(stride)) + return shape.as_int64() * stride.as_int64(); + int64_t bits = 0; + for (size_t i = 0; i < cutegen::rank(shape); ++i) { + auto extent = cutegen::get(shape, i); + auto step = cutegen::get(stride, i); + if (!cutegen::holds_int(extent) || !cutegen::holds_int(step)) + return std::nullopt; + bits = std::max(bits, extent.as_int64() * step.as_int64()); + } + return bits; +} + +LogicalResult +BitLayoutAttr::verify(function_ref emitError, + LayoutAttr layout) { + const cutegen::layout &l = layout.getRef(); + if (!cutegen::is_static(l)) + return emitError() << "bitlayout must be fully static but got " << layout; + if (cutegen::rank(l) != 2) + return emitError() << "bitlayout must have 2 modes corresponding to " + "(element bit, element index), but got " + << cutegen::rank(l); + auto elements = cutegen::size(l, 1); + if (cutegen::holds_int(elements) && elements.as_int64() == 0) + return emitError() + << "bitlayout must contain at least one logical element, but got 0"; + return success(); +} + +//===- ptr and memref -----------------------------------------------------===// + +namespace { +struct PtrFields { + Type valueType; + AddressSpace space = AddressSpace::generic; + std::optional alignment; + SwizzleAttr swizzle; + BitLayoutAttr bitlayout; +}; +} // namespace + +// `bitlayout<"layout">`, the keyword already read. +static BitLayoutAttr parseBitLayout(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text)) || + failed(parser.parseGreater())) + return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + parser.emitError(parser.getCurrentLocation(), "expects cg::layout"); + return {}; + } + MLIRContext *ctx = parser.getContext(); + return BitLayoutAttr::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, ctx, + LayoutAttr::get(ctx, std::move(*layout))); +} + +// `S`, the `S` already read. +static SwizzleAttr parseInlineSwizzle(AsmParser &parser) { + int64_t bits, base, shift; + if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || + failed(parser.parseComma()) || failed(parser.parseInteger(base)) || + failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || + failed(parser.parseGreater())) + return {}; + auto value = cutegen::from_string( + llvm::formatv("S<{0},{1},{2}>", bits, base, shift).str()); + if (!value) { + parser.emitError(parser.getCurrentLocation(), "invalid swizzle parameters"); + return {}; + } + return SwizzleAttr::get(parser.getContext(), std::move(*value)); +} + +// A layout or composed layout spelled as a string; a composed layout that is +// a plain one (identity swizzle, no offset) is read as that layout. +static Type parseLayoutPayload(AsmParser &parser) { + std::string text; + llvm::SMLoc loc = parser.getCurrentLocation(); + if (failed(parser.parseString(&text))) + return {}; + MLIRContext *ctx = parser.getContext(); + auto emitError = [&] { return parser.emitError(loc); }; + if (auto value = cutegen::from_string(text)) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, std::move(*value))); + if (auto value = cutegen::from_string(text)) { + if (value->is_normal_layout()) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, value->layout_b())); + return ComposedLayoutType::getChecked( + emitError, ctx, ComposedLayoutAttr::get(ctx, std::move(*value))); + } + parser.emitError(parser.getCurrentLocation(), "layout parse failure"); + return {}; +} + +static void printLayoutPayload(AsmPrinter &printer, Type layout) { + if (auto plain = llvm::dyn_cast(layout)) + printer.printString(cutegen::to_string(plain.getRef())); + else + printer.printString( + cutegen::to_string(llvm::cast(layout).getRef())); +} + +static LogicalResult memrefLayoutExpected(AsmParser &parser) { + return parser.emitError(parser.getCurrentLocation(), + "failed to parse CuteMemRefType parameter `,` " + "expected, layout cannot be omitted."); +} + +// `[type, ][space][, align][, S][, bitlayout<"L">]` in this order; +// a memref requires the type and ends with its layout string instead of the +// bit layout, which follows the layout. +static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { + PtrFields fields; + OptionalParseResult typed = parser.parseOptionalType(fields.valueType); + if (typed.has_value() && failed(*typed)) + return failure(); + if (layout && !typed.has_value()) + return memrefLayoutExpected(parser); + enum { kSpace, kAlign, kSwizzle, kBitLayout, kDone } next = kSpace; + bool first = !typed.has_value(); + while (true) { + if (!first && failed(parser.parseOptionalComma())) + break; + StringRef keyword; + if (failed(parser.parseOptionalKeyword(&keyword))) { + if (layout) { + if (!(*layout = parseLayoutPayload(parser))) + return failure(); + return fields; + } + if (first) + break; + parser.emitError(parser.getCurrentLocation(), + "expected a pointer field after `,`"); + return failure(); + } + first = false; + if (auto space = symbolizeAddressSpace(keyword); space && next <= kSpace) { + fields.space = *space; + next = kAlign; + } else if (keyword == "align" && next <= kAlign) { + uint64_t alignment; + if (failed(parser.parseLess()) || + failed(parser.parseInteger(alignment)) || + failed(parser.parseGreater())) + return failure(); + fields.alignment = alignment; + next = kSwizzle; + } else if (keyword == "S" && next <= kSwizzle) { + if (!(fields.swizzle = parseInlineSwizzle(parser))) + return failure(); + next = kBitLayout; + } else if (keyword == "bitlayout" && !layout && next <= kBitLayout) { + if (!(fields.bitlayout = parseBitLayout(parser))) + return failure(); + next = kDone; + } else { + parser.emitError(parser.getCurrentLocation(), + "unexpected pointer field `") + << keyword << "`"; + return failure(); + } + } + if (layout) + return memrefLayoutExpected(parser); + return fields; +} + +// Defaults are left out: they are what the parser fills in. +static PtrType buildPtr(AsmParser &parser, const PtrFields &fields) { + SwizzleAttr swizzle = + isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() : fields.swizzle; + BitLayoutAttr bitlayout = fields.bitlayout; + if (bitlayout && + bitlayout == PtrType::getDefaultBitLayout(fields.valueType, fields.space)) + bitlayout = {}; + return PtrType::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), fields.valueType, fields.space, + fields.alignment.value_or(PtrType::getNaturalAlignment(fields.valueType)), + swizzle, bitlayout); +} + +static void printPtrFields(AsmPrinter &printer, PtrType ptr) { + if (ptr.getValueType()) { + printer.printType(ptr.getValueType()); + printer << ", "; + } + printer << stringifyAddressSpace(ptr.getAddressSpace()); + if (ptr.getAlignment() != PtrType::getNaturalAlignment(ptr.getValueType())) + printer << ", align<" << ptr.getAlignment() << '>'; + if (!isIdentitySwizzle(ptr.getSwizzle())) + printer << ", " << cutegen::to_string(ptr.getSwizzle().getRef()); +} + +static void printBitLayout(AsmPrinter &printer, PtrType ptr) { + BitLayoutAttr bitlayout = ptr.getBitlayout(); + if (!bitlayout || bitlayout == PtrType::getDefaultBitLayout( + ptr.getValueType(), ptr.getAddressSpace())) + return; + printer << ", bitlayout<"; + printer.printString(cutegen::to_string(bitlayout.getLayout().getRef())); + printer << '>'; +} + +Type PtrType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + FailureOr fields = parsePtrFields(parser, nullptr); + if (failed(fields) || failed(parser.parseGreater())) + return {}; + return buildPtr(parser, *fields); +} + +void PtrType::print(AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, *this); + printBitLayout(printer, *this); + printer << '>'; +} + +LogicalResult PtrType::verify(function_ref emitError, + Type valueType, AddressSpace addressSpace, + uint64_t alignment, SwizzleAttr swizzle, + BitLayoutAttr bitlayout) { + if (alignment == 0 && getNaturalAlignment(valueType) != 0) + return emitError() << "expects alignment >= 1, got 0"; + bool tmem = addressSpace == AddressSpace::tmem; + if (tmem && swizzle && swizzle.getRef().num_bits() != 0) + return emitError() << "tmem pointer does not support non-identity swizzle"; + BitLayoutAttr defaultBitLayout = getDefaultBitLayout(valueType, addressSpace); + BitLayoutAttr effective = bitlayout ? bitlayout : defaultBitLayout; + if (!effective) + return success(); + if (auto bits = chunkBits(effective.getLayout().getRef())) { + if (*bits % 8 != 0) + return emitError() << "bitlayout chunk must be byte aligned, but size is " + << *bits << " bits"; + if (tmem && (*bits < 32 || !llvm::isPowerOf2_64(*bits))) + return emitError() + << "tmem pointer only supports 32 bit chunks that is a power of " + "2, got " + << *bits; + } + if (llvm::isa_and_nonnull(valueType) && + effective != defaultBitLayout) + return emitError() << "expects bitlayout to be " << defaultBitLayout + << " for valueType '" << valueType << "'but got " + << effective; + return success(); +} + +// Whether the offsets of `layout` are whole aligned blocks of `sparsity` +// elements: the contiguous run from offset 0 (stride-1 mode, then each mode +// whose stride is the run so far) is a multiple of it, and so is every other +// stride. +static bool isDenseForSparsity(const cutegen::layout &layout, + int64_t sparsity) { + if (!cutegen::is_static(layout)) + return false; + auto shape = cutegen::flatten(layout.shape()); + auto stride = cutegen::flatten(layout.stride()); + std::vector> modes; // (stride, extent) + auto add = [&](const auto &extent, const auto &step) { + if (extent.as_int64() != 1 && step.as_int64() != 0) + modes.emplace_back(step.as_int64(), extent.as_int64()); + }; + if (cutegen::holds_int(shape)) { + add(shape, stride); + } else { + for (size_t i = 0; i < cutegen::rank(shape); ++i) + add(cutegen::get(shape, i), cutegen::get(stride, i)); + } + llvm::sort(modes, [](auto a, auto b) { + return std::abs(a.first) < std::abs(b.first); + }); + int64_t run = 1; + bool contiguous = true; + for (auto [step, extent] : modes) { + if (contiguous && step == run) { + run *= extent; + continue; + } + contiguous = false; + if (step % sparsity != 0) + return false; + } + return run % sparsity == 0; +} + +Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + Type layout; + FailureOr fields = parsePtrFields(parser, &layout); + if (failed(fields)) + return {}; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("bitlayout")) || + !(fields->bitlayout = parseBitLayout(parser))) + return {}; + } + if (failed(parser.parseGreater())) + return {}; + PtrType ptr = buildPtr(parser, *fields); + if (!ptr) + return {}; + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), ptr, layout); +} + +void mlir::cutlass_compiler::cute::MemRefType::print( + AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, getPtr()); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printBitLayout(printer, getPtr()); + printer << '>'; +} + +LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( + function_ref emitError, PtrType ptr, Type layout) { + if (!ptr) + return emitError() << "expects a CuTe pointer"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + Type valueType = ptr.getValueType(); + if (!valueType || !(valueType.isIntOrIndexOrFloat() || + llvm::isa(valueType))) + return emitError() << "expects value type to be int, float, ptr or " + "sparse_elem, but got '" + << valueType << "'"; + if (auto sparse = llvm::dyn_cast(valueType)) { + int64_t sparsity = sparse.getNumLogical(); + bool dense; + if (auto composed = llvm::dyn_cast(layout)) { + // A swizzle may not move elements within a block. + const auto &c = composed.getRef(); + dense = c.is_a_swizzle() && + (int64_t(1) << c.swizzle_a().num_base()) % sparsity == 0 && + cutegen::holds_int(c.offset()) && + c.offset().as_int64() % sparsity == 0 && + isDenseForSparsity(c.layout_b(), sparsity); + } else { + dense = + isDenseForSparsity(llvm::cast(layout).getRef(), sparsity); + } + if (!dense) + return emitError() + << "layout must be dense but cannot be divided by sparsity"; + } + if (auto composed = llvm::dyn_cast(layout)) { + if (!isIdentitySwizzle(ptr.getSwizzle()) && + composed.getRef().is_a_swizzle() && + !composed.getRef().swizzle_a().is_identity()) + return emitError() << "cannot have both a swizzle on the pointer and a " + "non-identity swizzle in the layout"; + } else if (cutegen::has_scaled_basis( + llvm::cast(layout).getRef().stride())) { + return emitError() << "expected iterator and layout codomain to have " + "congruent profiles, but got iterator '" + << ptr << "' and layout layout: " + << cutegen::to_string( + llvm::cast(layout).getRef()); + } + return success(); +} + +//===- arith_tuple_iter, coord_tensor, i, fast_divmod_divisor, ... -----===// + +Type ArithTupleIteratorType::parse(AsmParser &parser) { + auto tuple = IntTupleType::parse(parser); + return tuple ? get(parser.getContext(), llvm::cast(tuple)) + : Type(); +} + +void ArithTupleIteratorType::print(AsmPrinter &printer) const { + getArithTuple().print(printer); +} + +Type CoordTensorType::parse(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text))) + return {}; + auto tuple = cutegen::from_string(text); + if (!tuple) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse CoordTensorType parameter 'arith_tuple' " + "which is to be a `cutegen::int_tuple`"); + return {}; + } + Type layout; + if (failed(parser.parseComma()) || !(layout = parseLayoutPayload(parser)) || + failed(parser.parseGreater())) + return {}; + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), + IntTupleType::get(parser.getContext(), std::move(*tuple)), layout); +} + +void CoordTensorType::print(AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getArithTuple().getRef())); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printer << '>'; +} + +LogicalResult +CoordTensorType::verify(function_ref emitError, + IntTupleType arithTuple, Type layout) { + if (!arithTuple) + return emitError() << "expects an arithmetic tuple"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + return success(); +} + +// `<[pow2, ]divby D>`, or nothing for a plain integer. +Type ConstrainedIntType::parse(AsmParser &parser, unsigned width) { + int64_t divisibility = 1; + bool isPow2 = false; + if (succeeded(parser.parseOptionalLess())) { + isPow2 = succeeded(parser.parseOptionalKeyword("pow2")); + bool divided = !isPow2 || succeeded(parser.parseOptionalComma()); + if (divided && (failed(parser.parseKeyword("divby")) || + failed(parser.parseInteger(divisibility)))) + return {}; + if (failed(parser.parseGreater())) + return {}; + } + return get(parser.getContext(), divisibility, width, isPow2); +} + +void ConstrainedIntType::print(AsmPrinter &printer) const { + printer << 'i' << getWidth(); + if (!getIsPow2() && getDivisibility() == 1) + return; + printer << '<'; + if (getIsPow2()) + printer << "pow2"; + if (getDivisibility() != 1) + printer << (getIsPow2() ? ", " : "") << "divby " << getDivisibility(); + printer << '>'; +} + +Type FastDivmodDivisorType::parse(AsmParser &parser) { + unsigned width; + if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) + return {}; + bool isPow2 = false; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("is_pow_2"))) + return {}; + isPow2 = true; + } + if (failed(parser.parseGreater())) + return {}; + return get(parser.getContext(), width, isPow2); +} + +void FastDivmodDivisorType::print(AsmPrinter &printer) const { + printer << '<' << getWidth(); + if (getIsPow_2()) + printer << ", is_pow_2"; + printer << '>'; +} + +LogicalResult +SparseElemType::verify(function_ref emitError, + int numLogical, Type physicalType) { + if (numLogical <= 0) + return emitError() << "sparse element type logical count must be > 0"; + Type element = physicalType; + if (auto vector = llvm::dyn_cast_or_null(physicalType)) + element = vector.getElementType(); + if (!llvm::isa_and_nonnull(element)) + return emitError() << "unsupported physical element type '" << physicalType + << "'"; + return success(); +} + +LogicalResult TiledMmaType::verify(function_ref emitError, + Type mmaAtom, LayoutAttr atomLayoutMNK, + TileAttr permutationMNK) { + if (cutegen::rank(atomLayoutMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 atomLayoutMNK"; + if (permutationMNK && cutegen::rank(permutationMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 permutationMNK"; + return success(); +} diff --git a/mlir/dialects/cute_ir/CuteTypesAll.td b/mlir/dialects/cute_ir/CuteTypesAll.td new file mode 100644 index 00000000..99766dcf --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypesAll.td @@ -0,0 +1,12 @@ +//===- CuteTypesAll.td - all cute types --------------------*- tablegen -*-===// +// +// The dialect has one generated type parser and printer, so the release's +// types and the compiler's are generated together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "mlir/dialects/cute_ir/CuteTypesCompiler.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD diff --git a/mlir/dialects/cute_ir/CuteTypesCompiler.td b/mlir/dialects/cute_ir/CuteTypesCompiler.td new file mode 100644 index 00000000..6ca5588a --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypesCompiler.td @@ -0,0 +1,269 @@ +// The cute dialect's types beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give the classes with a declarative syntax, their parameters in +// declaration order and their C++ types; the typed Python bindings of the +// types (`get` signatures and getters) give the parameters of the others; the +// live compiler, loaded from Python, gives the syntax (what it prints for what +// it parses, including every default it leaves out) and the verifier +// messages. +// +// Differences from the compiler, where its C++ types are not defined here: +// - The atoms of tiled_copy / tiled_copy_v2 / tiled_mma are +// `CopyAtomTypeInterface` / `CopyAtomTypeInterfaceV2` / +// `MmaAtomTypeInterface` in the compiler; plain types here. +// - ptr, memref and i have bespoke parsers in the compiler, so their +// parameter names and order are not recorded; the order here follows the +// Python `get` of each. +// - coord_tensor and memref layouts are `cutegen::layout` in the compiler's +// messages but accept composed layouts; they are a layout or composed +// layout type here. +// Verifier rules not reproduced: the bitlayout's cosize and element-count +// rules and TiledMMA's "requires static permutations of MNK" (neither +// triggered by any probe). The memref's sparse density rule ("layout must be +// dense but cannot be divided by sparsity") is inferred from probes: every +// probe agrees with it. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { + let summary = "Pointer with address space, alignment, swizzle and bit layout"; + let description = [{ + `<[value_type, ][space][, align][, S][, bitlayout<"L">]>`, the + fields in this order. Defaults are left out when printing and filled in + when parsing: + + - the address space is `generic` (always printed); + - the alignment is the value type's natural one: its width in bytes for + an integer or float that is a whole number of bytes, 1 for `i1`, 0 + (none) for other sub-byte widths, the physical type's bytes for a + `!cute.sparse_elem`, and 1 for any other type or no type; + - the swizzle is the identity `S<0,4,3>`; + - the bit layout is `(b,n):(1,s)` for an integer or float of `b` bits + (`s` = 8 for `i1`, else `b`) with `n` the least count making a chunk of + whole bytes (32-bit chunks in `tmem`), and + `(pb,(L,n)):(1,(0,pb))` for a `!cute.sparse_elem` of `pb` bits. + + A sparse pointer (the Python `sparsity`) is a `!cute.sparse_elem` value + type. + + ```mlir + !cute.ptr, S<2,4,3>> + !cute.ptr + !cute.ptr> + ``` + }]; + let parameters = (ins + Cute_OptionalParam<"mlir::Type", "pointee type">:$value_type, + Cute_Param<"AddressSpace", "address space">:$address_space, + Cute_Param<"uint64_t", "alignment in bytes; 0 only as a sub-byte type's natural alignment">:$alignment, + Cute_OptionalParam<"SwizzleAttr", "swizzle; null for the identity">:$swizzle, + Cute_OptionalParam<"BitLayoutAttr", "bit layout; null for the default">:$bitlayout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + /// The alignment a pointer to `valueType` has when none is spelled. + static uint64_t getNaturalAlignment(mlir::Type valueType); + /// The bit layout a pointer has when none is spelled (null if none). + static BitLayoutAttr getDefaultBitLayout(mlir::Type valueType, + AddressSpace addressSpace); + }]; +} + +def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { + let summary = "Tensor view: a pointer with a layout"; + let description = [{ + `][, S], "layout"[, bitlayout<"L">]>`: + the pointer fields as in `!cute.ptr` (the value type is required), then the + layout or composed layout, then a non-default bit layout. A composed + layout whose inner part is the identity swizzle is read as its outer + layout. + + ```mlir + !cute.memref, S<2,4,3>, "((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))"> + !cute.memref> + ``` + }]; + let parameters = (ins + Cute_Param<"PtrType", "pointer">:$ptr, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + mlir::Type getValueType() const { return getPtr().getValueType(); } + AddressSpace getAddressSpace() const { return getPtr().getAddressSpace(); } + uint64_t getAlignment() const { return getPtr().getAlignment(); } + SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } + }]; +} + +def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { + let summary = "Coordinate tensor: an arithmetic tuple with a layout"; + let description = [{ + ```mlir + !cute.coord_tensor<"(?,?,?)", "((512,1),1):((1/2@0,0),0)"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "origin arithmetic tuple">:$arith_tuple, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_ArithTupleIteratorType + : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { + let summary = "Iterator over an arithmetic tuple"; + let description = [{ + ```mlir + !cute.arith_tuple_iter<"(?,?{div=16})"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "current arithmetic tuple">:$arith_tuple + ); + // `$arith_tuple` alone would print a space after the mnemonic. + let hasCustomAssemblyFormat = 1; +} + +// `!cute.i`: one mnemonic per width, which the generated type dispatch +// cannot match, so the class has no mnemonic and CuteTypes.cpp routes +// `i` keywords to it. +def Cute_ConstrainedIntType : TypeDef { + let summary = "Integer of any width with known divisibility and power of two"; + let description = [{ + `!cute.i[<[pow2, ]divby D>]`; `divby 1` is left out (the default), + `divby 0` and negative divisors are kept. The compiler also reads + `!cute_nvgpu.i` as this type. + + ```mlir + !cute.i32 + !cute.i16 + !cute.i64 + ``` + }]; + // The compiler's type name for it. + let typeName = "cute.ConstrainedInt"; + let parameters = (ins + Cute_Param<"int64_t", "known divisor">:$divisibility, + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "known to be a power of two">:$is_pow2 + ); + let extraClassDeclaration = [{ + static mlir::Type parse(mlir::AsmParser &parser, unsigned width); + void print(mlir::AsmPrinter &printer) const; + }]; +} + +def Cute_FastDivmodDivisorType + : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { + let summary = "Precomputed divisor of the fast divmod operations"; + let description = [{ + ```mlir + !cute.fast_divmod_divisor<32> + !cute.fast_divmod_divisor<64, is_pow_2> + ``` + }]; + let parameters = (ins + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "whether the divisor is a power of two">:$is_pow_2 + ); + let hasCustomAssemblyFormat = 1; +} + +def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { + let summary = "Logical sparse element stored in a physical type"; + let description = [{ + `num_logical` logical elements stored as `physical_type` (an integer, + float or vector of them). + + ```mlir + !cute.sparse_elem<2, vector<1xf16>> + ``` + }]; + let parameters = (ins + Cute_Param<"int", "number of logical elements">:$num_logical, + Cute_Param<"mlir::Type", "physical storage type">:$physical_type + ); + let assemblyFormat = "`<` $num_logical `,` $physical_type `>`"; + let genVerifyDecl = 1; +} + +def Cute_TiledCopyType : Cute_Type<"TiledCopy", "tiled_copy"> { + let summary = "Copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy, layout_copy_tv = <"(32,4):(4,1)">, tiler_mn = <"[4:1;32:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledCopyV2Type : Cute_Type<"TiledCopyV2", "tiled_copy_v2"> { + let summary = "V2 copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy_v2, layout_copy_tv = <"(1,8):(0,1)">, tiler_mn = <"[8:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "V2 copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledMmaType : Cute_Type<"TiledMma", "tiled_mma"> { + let summary = "MMA atom tiled over MNK"; + let description = [{ + The permutation is optional (absent, not `[_;_;_]`, when not given). + + ```mlir + !cute.tiled_mma f32 >, atom_layout_MNK = <"(16,16,1):(16,1,0)">, permutation_MNK = <"[(16,4):(4,1);(16,4):(4,1);_]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "mma atom">:$mma_atom, + Cute_Param<"LayoutAttr", "rank-3 atom layout over MNK">:$atom_layout_MNK, + Cute_OptionalParam<"TileAttr", "rank-3 permutation over MNK">:$permutation_MNK + ); + let assemblyFormat = [{ + `<` $mma_atom `,` `atom_layout_MNK` `=` $atom_layout_MNK + (`,` `permutation_MNK` `=` $permutation_MNK^)? `>` + }]; + let genVerifyDecl = 1; +} + +def Cute_TupleType : Cute_Type<"Tuple", "tuple"> { + let summary = "Non-empty tuple of types"; + let description = [{ + ```mlir + !cute.tuple + ``` + }]; + let parameters = (ins ArrayRefParameter<"mlir::Type", "element types">:$types); + let assemblyFormat = "`<` $types `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md new file mode 100644 index 00000000..6a667ce5 --- /dev/null +++ b/mlir/dialects/cute_ir/README.md @@ -0,0 +1,106 @@ +# CuTe dialects and Zig bindings + +`mlir/dialects/cute_ir` is the CuTe dialect of nvidia-cutlass-dsl 4.8.0. The +dialect, attributes, public types and `cutegen` layout algebra come verbatim +from the CUTLASS release's `cutlass_compiler/` tree (`@cute_ir`, +pinned in `third_party/cute_ir`, one patch giving cutegen the +`?{div=N}` leaf); this tree carries what differs from it: `CuteTypes.cpp` with +the compiler's remaining types, attributes and enums and ratio strides +allowed, the compiler's attributes, types and enums beyond the release +(`CuteAttrsCompiler.td`, `CuteTypesCompiler.td`, `CuteEnums.td` and the +`../cute_nvgpu` definitions), and every operation of the compiler's registry in +`CuteOps.td` and the companion `CuteNVGPUOps.td`. + +The companion dialects live in `../cute_nvgpu`, `../nvvm`, and `../cuda`, +with separate Bazel targets and Zig modules. NVIDIA's `nvvm` differs from +upstream's. The CuTe builder registers these dialects together with upstream +`llvm`. + +Updating to a new DSL release means refreshing the compiler entry-point +offsets and regenerating the `.td` files from the release's Python bindings +and compiler. The C API (except the algebra attribute functions in `capi.cc`) +and the Zig types, attributes, enums and operation builders are generated +from those `.td` files and follow them. + +## Zig + +Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`. +`cute.registerDialect(registry)` registers `cute`. + +Import each companion dialect explicitly and add its Bazel dependency: +`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and `mlir/dialects/cuda`. +CUDA and NVVM expose generated bindings in `.ops` and convenience builders +at the module root. NVIDIA's NVVM handle is `cute_nvvm`; its dialect cannot +share a registry with upstream NVVM. + +```zig +const cute = @import("mlir/dialects/cute_ir"); + +const pointer = try cute.PtrType.get(ctx, .{ + .valueType = .float(ctx, .f32), + .addressSpace = .smem, + .alignment = 16, +}); +const layout_attr = try cute.LayoutAttr.get(ctx, .{ + .shape = try .getStatic(ctx, 4), + .stride = try .getStatic(ctx, 1), +}); +const layout = try cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() }); +const memref = try cute.MemRefType.get(ctx, .{ + .ptr = pointer.type_(), + .layout = layout.type_(), +}); +const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); +``` + +The bindings follow `mlir/dialects/README.md`. Algebra types take an algebra +attribute, `cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() })`. +Algebra attributes are built bottom up with named constructors: integer +tuples, coordinates, shapes and strides from `getStatic`, `getDynamic`, +`getBasis` (strides) or `getUnderscore` (coordinates) leaves and `getTuple`; +layouts from a shape and a stride; tiles from `getLayout`, `getUnderscore` and +`getTuple`; swizzles from their three parameters and composed layouts from an +inner swizzle or layout, an offset and an outer layout. `value()` returns the +canonical text. NVGPU atom types take their typed parameters, such +as `.valType` and `.copyBits`. For reading existing IR use MLIR's generic +`Type.parse`. + +Operation builders take the operands, then the result types, then the +attributes, then the location, and call `mlir.Operation.make`. Parameters +follow ODS names, with a trailing underscore where needed. Result parameters +append `_type` or `_types`; attributes take `*const mlir.Attribute`. Optional +operands, attributes and result types go in a trailing `opts` struct whose +fields default to null; pass `.{}` to omit them, and `&.{}` for empty variadic +operands. Dots in +mnemonics become underscores. Variable-length operand and result groups go +through `Operation.make`'s `.variadic` API, which writes the segment-size +attributes. Builders verify the created operations; to defer verification, use +`mlir.Operation.make` directly with `.verify = false`. + +Builders of operations with regions (`cuda.kernel`) and of terminators +(`cuda.return`) do not verify on creation, since their blocks are completed +afterwards; the module is verified instead. + +Each dialect exposes a `:capi` target and a package-local `capi.h`, independently +of Zig: `//mlir/dialects/cute_ir:capi`, `//mlir/dialects/cute_nvgpu:capi`, +`//mlir/dialects/nvvm:capi`, and `//mlir/dialects/cuda:capi`. C API declarations +and definitions are grouped by registration, attributes, and types in +`capi.h` and `capi.cc`. Constructors return a null handle for invalid parameters. + +C++ implementations, headers, and TableGen sources live directly in each +dialect package. Generated `.inc` files are emitted into the same package +under Bazel's output tree. Virtual headers preserve upstream CuTe include +paths required by the release sources. + +```sh +bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_nvgpu:test \ + //mlir/dialects/nvvm:test //mlir/dialects/cuda:test +``` + +These tests only check the bindings: every bound operation is registered, +every type and attribute rebuilds from its getters, and a sample of the nvvm +and cuda attributes and types print the compiler's own text byte for byte +(`#cuda` and `#cuda` print with a +space the compiler omits; it parses both). The dialect itself is +tested where it is maintained. The dialects here do not implement the +compiler's semantic verification or lowering. diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md new file mode 100644 index 00000000..52b49e69 --- /dev/null +++ b/mlir/dialects/cute_ir/REGENERATING.md @@ -0,0 +1,333 @@ +# Regenerating the dialect definitions + +These are instructions for an agent. They describe what this directory must +contain after a CuTe DSL release bump, what is available to derive it, and how +to check the result. How to get there is left to you: combine the sources as +you see fit, write throwaway scripts, probe the compiler. Scripts and +intermediate tables are not committed; only the definitions are. + +Bare file names are in this directory, `../` are the sibling packages, +and other paths are relative to the repository root. + +## Goal + +The `cute` dialect in this directory and its companions in `../cute_nvgpu`, +`../nvvm`, and `../cuda` must describe the DSL compiler's +registry as completely and as precisely as the release allows: every +operation, type, attribute and enum the compiler registers, with every fact +that can be established about each of them. Nothing is opaque unless it cannot +be recovered, and each remaining gap is stated in the file where it occurs. + +The DSL compiler stays the authority for semantics and lowering. These +definitions exist so that IR can be built, parsed, printed and type-inferred +without it, and so that anything built here is accepted by it unchanged. + +## What lives here + +- **Taken from the release archive.** Its `cutlass_compiler/` tree provides the + cute dialect declaration, the public types and attributes and their `.td` + files, the type interfaces, `CuteDialect.h`, `CuteAttributes.cpp`, + `CuteDialect.cpp` and the `cutegen` layout algebra. The build reads them + from the archive; they are never copied or edited here. One patch on the + archive (`cutegen_dynamic_divisibility.patch`, next to the archive's + repository rule) teaches `cutegen` the `?{div=N}` leaf. Refresh it if it no + longer applies. The archive's `CuteOps.td` is not used: its operations + differ from the compiler's. +- **Maintained here.** `CuteTypes.cpp` started as the release's copy and holds + the parsers, printers and verifiers of every cute type, fixed wherever the + release's parsing diverges from the compiler's. `CuteInference.{h,cpp}` holds + result type inference; the other dialect `.cpp` files and the `BUILD.bazel` + files complete them. Edit them as the new definitions require. For `nvvm` + and `cuda` these are `NVVMDialect.{h,cpp}` (with the `#nvvm.target` + verifier) and `CudaDialect.{h,cpp}` in their dialect package roots. +- **Generated: what you produce.** Every package-local `.td` file that is not + a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, + `NVVMOps.td`, `CudaOps.td`), the compiler's types, attributes and enums + beyond the release (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, + `CuteEnums.td`, `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, + `NVVMAttrs.td`, `NVVMEnums.td`, `CudaTypes.td`, `CudaAttrs.td`, + `CudaEnums.td`), the inference of `nvvm` and `cuda` operations whose results + are not one buildable type (`NVVMInference.cpp`, `CudaInference.cpp`), and + any new `.td` you need to split them sensibly. The dialect declarations (`*Dialect.td`) change only as far as the + generated files need. +- **Language bindings, if present.** When the tree carries bindings derived + from these definitions (a `capi.cc`/`capi.h` C API in each dialect package, builder files in each + directory), regenerate them from the records of the new definitions + (`llvm-tblgen --dump-json` gives them all; do not regex the `.td` files). + When a class changes shape (opaque to parametric, merged, split, renamed + accessors), change the bindings to match, and keep the existing + convenience helpers working where it costs little. Names, builder + parameters and `opts` follow `mlir/dialects/README.md`. + +Consumers outside this directory use the dialect classes and must keep +compiling: search the repository for the dialect namespaces +(`cutlass_compiler::cute`, `cute_nvgpu`, `nvvm`, `cuda`), for the Zig modules +(`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and +`mlir/dialects/cuda`) and for the headers +in the dialect packages. When a definition changes shape, update those users in the +same change. Tighter operand constraints also break consumers that built IR +with loose types, and so does an operation that stops inferring its results: +fix the consumer, not the constraint. + +## NVIDIA's `nvvm` and `cuda` dialects + +The kernels the CuTe builder emits (`dsl/cute`) use two more dialects of +the compiler: `nvvm` (special registers, mbarriers, tcgen05, ...) and `cuda` +(kernels, launch configurations, host runtime calls). Both have sibling packages +and must be regenerated with every release, like `cute` and `cute_nvgpu`. + +- **Neither is upstream's.** The compiler is built on NVIDIA's own LLVM + revision (the `.so` names it: `LLVM version 23.0.0git ()`, a hash + upstream does not have), and its `nvvm` differs from upstream NVVM: it has + operations, operands and enums upstream lacks (`nvvm.mbarrier.wait.parity`, + `#nvvm.mbar_wait`) and lacks some upstream has. `cuda` exists only in the + compiler. No `.td` ships with the release for either: rebuild both from the + bindings and the live compiler, as for the cute dialects. Upstream's NVVM + `.td` is neither a source nor a substitute. +- **Where they live.** `../nvvm` and `../cuda` contain the declarations, + generated `.td` files, dialect and inference `.cpp` files, and consolidated + `capi.h`/`capi.cc` C APIs. Each package's `ops.zig` contains the generated + bindings; `nvvm.zig` and `cuda.zig` provide the convenience builders. + +- **Names.** The C++ namespaces are `mlir::cutlass_compiler::nvvm` and + `mlir::cutlass_compiler::cuda`. The dialects are named `nvvm` and `cuda` as + in the compiler, so a registry holds this `nvvm` or upstream's, never both; + upstream's NVVM C++ is linked into the tree and must stay unregistered where + this one is. The C dialect handles are `cute_nvvm` and `cuda`, and the C API + of `nvvm` is prefixed `CuteNVVM` so that neither collides with upstream's + `mlirGetDialectHandle__nvvm__` or a future upstream NVVM C API; `cuda`'s is + prefixed `Cuda`. +- **`llvm` is upstream's.** NVVM operands are `!llvm.ptr` and LLVM + structs, and the builder emits `llvm.*` operations. The CuTe context + registers the upstream LLVM dialect, bound by `mlir/dialects/llvm/llvm.zig` + (its builders, and `mlirLLVMPointerTypeGet` for `!llvm.ptr`); the nvvm + and cuda constraints name `::mlir::LLVM` types. NVIDIA's `llvm` is not + vendored: its types print and parse the same, and the compiler reads the + `llvm.*` operations upstream prints (custom and generic forms, with the + properties upstream fills in) into the same module. This holds only while + the compatibility check below passes; when it stops holding, vendor NVIDIA's + `llvm` the same way (its bindings are `_llvm_ops_gen.py` and + `_llvm_enum_gen.py`) rather than patching around it. + +Sources specific to them, besides those in the table below: + +- `_nvvm_ops_gen.py`, `_cuda_ops_gen.py`: the operations. Operand kinds come + from how `__init__` appends them (and `_ODS_OPERAND_SEGMENTS`), result names + and kinds from the result properties, inference from whether `__init__` + builds a `results` list or passes its `results=` argument through. +- `_nvvm_enum_gen.py`, `_cuda_enum_gen.py`: the enums. A builder registered + under the enum's own name builds the plain `I32EnumAttr` (it follows its + enum class in the file); one registered as `nvvm.Attr` builds the + dialect attribute and gives its mnemonic and form: `#nvvm.` + or `#nvvm`. Some enums only have the latter. Parsing an + invalid case names the C++ enum (`expected ::mlir::NVVM:: to be one + of: ...`) or its summary, which maps each attribute to its enum. +- `cutlass._mlir._mlir_libs._cutlass_ir._mlirDialectsCuda`: the typed cuda + type and attribute classes and their `get()` parameters. The other cuda + types and attributes, and the three non-enum nvvm attributes (`shape`, + `ld_st_matrix_shape`, `target`), are found by trying the `.so`'s + identifiers as mnemonics (`!cuda.`, `#nvvm.`); `nvvm` has no + types. +- Attribute constraints the bindings name but no builder covers + (`nvvm.FPArithRoundingMode`, the `nvvm.Multimem*` orders, anonymous + `TypeAttr` constraints) are recovered by probing values: the verifier's + `failed to satisfy constraint: ...` names the rule, and the values it lets + through give the predicate. +- The compiler aborts on some malformed probes (`nvvm.mma.sync`, + `nvvm.mma.sp.sync`): run those checks one process each. +- Results the compiler infers but that are not one buildable type get an + inference rule only where varying the operand types shows the result is one + operand's type; elsewhere inference fails and the types written stand + (`_InferredOp`, as in cute). Vary the attributes too: a rule must + hold for every attribute value (`nvvm.mul` with `mode = wide` doubles the + width, `nvvm.shfl.sync` with `return_value_and_is_valid` returns a struct), + or inference must fail. + +The bindings follow the cute ones: the C API and the Zig enums, typed +attribute and type wrappers and operation builders are generated from +`llvm-tblgen --dump-json` of the generated `.td` files, following +`mlir/dialects/README.md` (names, wrappers, comments, formatting). Builders of +operations with regions, and of terminators, do not verify on creation (their +blocks are filled, or their block is completed, afterwards). `mlir/dialects/nvvm/nvvm.zig` and `mlir/dialects/cuda/cuda.zig` are the +hand-written convenience layer the builder calls; they fix the attributes the +kernels do not choose and must keep building no text. + +## Sources + +Pin everything to one release version, read from the repository rule that +fetches the release archive. The same version names the Python wheels. + +The packaging below is the one of the release these instructions were last +run against. Releases move things around: when a source is missing or has +changed shape, find where the same fact now lives, and update this file. + +| Source | Where | What it gives | +|---|---|---| +| Release archive | `https://github.com/NVIDIA/cutlass/archive/refs/tags/v.tar.gz`, `cutlass_compiler/` | The release's own `.td` files: what is already defined, their style, the parameter C++ types and helpers (`Cute_Param`, interfaces). Anything defined there is not redefined here. | +| Operation bindings | wheel `nvidia-cutlass-dsl-libs-base`, `cutlass/_mlir/dialects/__ops_gen.py` | Per operation: name, class name, docstring (summary, description, examples), operands and results in order with single, optional or variadic kind, segment attributes, attribute names with their ODS constraint (`AttrBuilder.get('')`), whether the builder infers result types, region count, successors. | +| Enum bindings | same wheel, `__enum_gen.py` | Every enum: cases, integer values, spellings, and for the ones that are attributes, the mnemonic they print under. | +| Compiler library | wheel `nvidia-cutlass-dsl-libs-cu13` (or `-cu12`), `cutlass/_mlir/_mlir_libs/_cutlass_ir.cu13.*.so` | Its strings: type and attribute class names, parameter names and C++ types (from the generated parser messages `failed to parse parameter '

' which is to be a `), mnemonics, verifier messages and constraint summaries. The strings are not in declaration order, and parameters read by custom directives have no message at all. | +| Typed Python classes | the same `.so`, as `cutlass._mlir.dialects.cute`, `cute_nvgpu`, `cuda` | For every compiler type and attribute: a `get(...)` with its parameters in order, and getters. Building a value and printing it shows the canonical syntax of each parameter value. | +| The examples | the release archive's `examples/python/CuTeDSL` | Real compiler-accepted modules, the round-trip corpus (see below). | +| The live compiler | the same `.so`, loaded from Python | Ground truth for everything above. | + +Loading the compiler. The `.so` only runs on Linux, on the architecture it was +built for. Two ways: + +- **Full install** (needed for the typed classes): a venv with + `nvidia-cutlass-dsl==` installed, then `import cutlass` before + anything else. +- **Bare wheels** (generic API only): unpack the two wheels into one directory, + put its `nvidia_cutlass_dsl/dsl_packages` on `sys.path`, `import + cutlass._mlir._mlir_libs`. The typed classes fail to import this way. + +Then use `cutlass._mlir.ir` with a fresh `ir.Context()`. The compiler +segfaults on some malformed inputs (some bitlayouts, some bad inference +inputs): run probes in a forked process. + +What the live compiler tells you: + +- `ir.Type.parse` / `ir.Attribute.parse` accept or reject a spelling, and on + rejection say where and what they expected; `str()` gives the canonical + printed form. Together with the typed classes this establishes each syntax: + the generated parser messages name each missing parameter, and the bespoke + custom directives (`128x128x32`, `32 DP`, `x32`, trailing flags such as + `mcast` or `t`) show up as `expected ...` errors. Every mnemonic a parser + knows can be found by trying every identifier-like string of the `.so`. +- `ir.Module.parse` on generic-form operations with wrong types yields the + verifier's constraint for each operand and result, in the compiler's words + (`operand #0 must be ... , but got ...`). Probe each operand with one + instance of every type class to learn exactly which classes it accepts. +- `op.has_trait(ir.IsTerminatorTrait)`, `ir.NoTerminatorTrait`; constructing + `ir.InferTypeOpInterface(op)` or `ir.MemoryEffectsOpInterface(op)` fails when + the operation lacks the interface. +- `ir.InferTypeOpInterface().inferReturnTypes(operands=..., + attributes=..., context=...)` is the inference oracle, with operand values + taken as block arguments of a dummy region. It cannot take properties: + for operations with attributes, parse and verify a module instead. +- Loop-invariant code motion hoists `Pure` operations, `canonicalize` removes + unused ones that do not write, and `cse` merges ones that at most read. +- Printing a parsed generic operation shows its custom assembly. + +The round-trip corpus: run every example of the archive's +`examples/python/CuTeDSL` with `CUTE_DSL_KEEP=ir-debug`, which keeps each +compiled module's IR; set `CUTE_DSL_ARCH` for the architecture-restricted +ones. This gives hundreds of real modules covering far more operations than +the kernels' fixtures in the repository, which are added to it. + +When sources disagree, the live compiler wins; record the disagreement in a +comment. + +## What each definition must carry + +Operations: + +- The operation name, and the binding's class name as the def name. +- `summary` and `description` from the docstring, kept whole, including its + examples. Reflow nothing that is code. Every example must parse and verify in + the compiler; fix the ones that do not and say so. +- Every operand and result, in order, with its name, its kind (single, + `Optional`, `Variadic`), and a type constraint that accepts exactly the type + classes the compiler's verifier accepts, with the verifier's wording as its + summary. Where a class or interface is not defined here or in the release, + or no probe type satisfies it, the predicate accepts any type and the file + says so. Constraints on types of unregistered dialects (such as `ptr`) must + also match the opaque form those types take here. A constraint must never + reject what the compiler accepts. +- Every attribute with its constraint: the ODS constraint the bindings name, + or the generated enum or dialect attribute it corresponds to; `OptionalAttr` + and `DefaultValuedAttr` as the bindings show. +- `AttrSizedOperandSegments` / `AttrSizedResultSegments`, regions (count and + kind), successors, terminator traits. +- `Pure` (or the memory effects) when established; none when unknown. +- `InferTypeOpInterface` exactly on the operations whose result types the + compiler infers. ODS generates the inference when all results are buildable + types; the others are written in `CuteInference.cpp`. The compiler's + inference is not a function of types alone: it folds constant operands and + keeps divisibility that `cutegen` drops. Where a rule is not recovered, + inference must fail rather than guess, and those operations override + `refineReturnTypes` so written result types stand: compiler-accepted IR is + never rejected for want of an inference rule. +- Operations keep the generic assembly form unless an `assemblyFormat` has been + checked to round-trip against the compiler (see below); the custom syntax + stays in the description. + +Types and attributes: + +- Every class the compiler has that the release lacks, with its mnemonic, + summary, parameters in declaration order (from the typed `get()`s and the + order parsing asks for them) with their C++ types and optional or defaulted + status, and a printed example in the description. C++ types of parameters + read by custom directives are not observable: choose the natural one. +- The syntax exactly as the compiler prints and parses it: a declarative + `assemblyFormat` when it can express it, otherwise a custom parser and + printer. Parametric beats opaque: a verbatim string payload is acceptable + only when the syntax cannot be reproduced, and the file says why. A hidden + parameter that never prints is left out: keeping it would make a built value + differ from its own printed text. +- The compiler's type aliases (`!memref_smem_f16`, `!copy_simt`, ...) are part + of its printed form: reproduce them through the dialect's + `OpAsmDialectInterface`. +- Verifiers (`genVerifyDecl`) where the compiler rejects parameter values, when + the rule is known. Large legality tables (instruction shapes and forms) may + be left to the compiler; the file says which. +- Enums as `I32EnumAttr` with the compiler's values and spellings, and their + `EnumAttr` with the compiler's mnemonic and `` form. + +Every generated file starts with a short header naming the release version and +the sources it came from, saying it is generated, and listing what it leaves +out or leaves unconstrained, and why. + +ODS pitfalls met so far: + +- Naming `InferTypeOpInterface` in the traits of an operation with buildable + results suppresses the `inferReturnTypes` ODS would generate. +- `mlir-tblgen -gen-enum-decls` emits every enum it sees, so a dialect's enums + file must not include another dialect's. +- The release's `CuteDialect.h` includes the attributes before the types, so + enums the attributes use must be emitted ahead of the attributes' include. +- A mnemonic that varies with a value (`!cute.i32`, `!cute.i7`) needs a hook + in front of the generated type parser. +- An attribute whose body is not in `<...>` prints in the opaque form + (`#dialect<...>`); its pretty form does not parse. + +## Checking the result + +A change is done when all of these hold: + +1. **Coverage.** Every operation in the bindings, every enum, and every + mnemonic the compiler's parsers know is either defined here, defined by the + release, or listed in the file's header as left out with the reason. +2. **Round trip against the compiler,** on the corpus and on sweeps of + spellings for every type and attribute (combinations of parameter values): + whatever the compiler accepts, this dialect accepts, and prints text the + compiler reads back to its own canonical form. Where the compiler's printed + form is itself reparseable, this dialect prints it byte-identical. Where + the compiler prints text its own parser rejects, or its parser drops data, + the reparseable, lossless form wins; list those cases. +3. **Inference agrees** with the compiler on every result the corpus and + crafted oracle cases produce. The inference test holds recorded compiler + results, since the compiler cannot be asked at test time: add a case for + every inferred operation the emitted modules do not already cover. +4. **The repository's tests pass**, including the dialect's own tests and the + tests of every consumer found above (kernel emitters, host tools, language + bindings), built with the tree's usual Bazel configuration. +5. **NVIDIA's compiler reads what the kernels emit.** Dump the IR of every + kernel the repository's tests emit through `dsl/cute`: the builder's + own tests and the CuTe kernels under `zml/kernels/moe/cute`, + `zml/kernels/flash_attention/cute` and `zml/kernels/paged_attention/cute`. + Dump both with the previous definitions and with the new ones. In the + release's venv, `import cutlass`, then parse and + verify each module with `cutlass._mlir.ir.Module.parse`, and compare the + generic print (`get_asm(print_generic_op_form=True)`) of each module before + and after: it must be identical, and a module the compiler rejected before + must be rejected the same way after. This is also what keeps upstream + `llvm` acceptable (see above). +6. **Both copies agree.** When the same dialect is carried by more than one + repository, the shared files are identical in each, and only + repository-specific files (`BUILD.bazel`, bindings) differ. + +Report, at the end: what changed in each file, the coverage figures, the +round-trip and inference figures, what is still opaque or unconstrained and +why, the test results, and anything in these instructions that was wrong, +missing or unclear. diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc new file mode 100644 index 00000000..6e7c296d --- /dev/null +++ b/mlir/dialects/cute_ir/capi.cc @@ -0,0 +1,1454 @@ +//===- capi.cc - C API for the cute dialect -------------------------------===// + +#include "capi.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "llvm/ADT/STLExtras.h" + +#include + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, + mlir::cutlass_compiler::cute::CuteDialect) + +// Attributes + +namespace cute = mlir::cutlass_compiler::cute; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +template +static MlirAttribute getAlgebraAttr(MlirContext context, + typename Attr::algebra_t algebra) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(Attr::get(ctx, std::move(algebra))); +} + +template +static MlirAttribute getStaticAttr(MlirContext context, int64_t value) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::static_int_t(value))); +} + +template +static MlirAttribute getDynamicAttr(MlirContext context, int32_t width, + int64_t divisibility) { + if (!llvm::is_contained({1, 8, 16, 32, 64}, width) || divisibility <= 0) + return {nullptr}; + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::mlir_dynamic_t( + cutegen::dynamic_int_properties_t(width, divisibility)))); +} + +template +static MlirAttribute getUnderscoreAttr(MlirContext context) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::underscore_t())); +} + +template +static MlirAttribute getTupleAttr(MlirContext context, intptr_t numElements, + MlirAttribute const *elements) { + auto *ctx = unwrap(context); + if (!ctx || numElements < 0 || (numElements > 0 && !elements)) + return {nullptr}; + typename Attr::algebra_t::vector_t children; + children.reserve(numElements); + for (intptr_t i = 0; i < numElements; ++i) { + Attr element; + if (!unwrapAs(ctx, elements[i], /*optional=*/false, element)) + return {nullptr}; + children.push_back(element.getRef()); + } + return getAlgebraAttr(context, + typename Attr::algebra_t(std::move(children))); +} + +template +static MlirStringRef getAlgebraValue(MlirAttribute attr) { + auto value = llvm::cast(unwrap(attr)); + // Intern the serialization so the returned view outlives this function. + auto text = mlir::StringAttr::get(value.getContext(), + cutegen::to_string(value.getRef())); + return wrap(text.getValue()); +} + +MlirAttribute mlirCuteIntTupleAttrStaticGet(MlirContext context, + int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteIntTupleAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteIntTupleAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteCoordAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteCoordAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteCoordAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteCoord(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteShapeAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteShapeAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { + return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); +} + +MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteStrideAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteStrideAttrBasisGet(MlirContext context, int64_t numerator, + int64_t denominator, intptr_t numModes, + int32_t const *modes) { + if (denominator <= 0 || numModes <= 0 || !modes) + return {nullptr}; + cutegen::scaled_basis::vector_t path(modes, modes + numModes); + auto basis = + denominator == 1 + ? cutegen::scaled_basis(std::move(path), numerator) + : cutegen::scaled_basis(std::move(path), + cutegen::ratio(numerator, denominator)); + return getAlgebraAttr(context, + cutegen::stride(std::move(basis))); +} + +MlirAttribute mlirCuteStrideAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteStride(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + auto *ctx = unwrap(context); + cute::ShapeAttr shapeAttr; + cute::StrideAttr strideAttr; + if (!ctx || !unwrapAs(ctx, shape, /*optional=*/false, shapeAttr) || + !unwrapAs(ctx, stride, /*optional=*/false, strideAttr)) + return {nullptr}; + cutegen::layout layout(shapeAttr.getRef(), strideAttr.getRef()); + if (cutegen::has_error(layout)) + return {nullptr}; + return getAlgebraAttr(context, std::move(layout)); +} + +bool mlirAttributeIsACuteLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteTileAttrLayoutGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + cute::LayoutAttr layoutAttr; + if (!ctx || !unwrapAs(ctx, layout, /*optional=*/false, layoutAttr)) + return {nullptr}; + return getAlgebraAttr(context, + cutegen::tile(layoutAttr.getRef())); +} + +MlirAttribute mlirCuteTileAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteTileAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteTile(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, + MlirAttribute inner, + MlirAttribute offset, + MlirAttribute outer) { + auto *ctx = unwrap(context); + cute::IntTupleAttr offsetAttr; + cute::LayoutAttr outerAttr; + if (!ctx || !unwrapAs(ctx, offset, /*optional=*/false, offsetAttr) || + !unwrapAs(ctx, outer, /*optional=*/false, outerAttr)) + return {nullptr}; + cute::SwizzleAttr swizzleAttr; + cute::LayoutAttr layoutAttr; + std::optional composed; + if (unwrapAs(ctx, inner, /*optional=*/false, swizzleAttr)) + composed.emplace(swizzleAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + else if (unwrapAs(ctx, inner, /*optional=*/false, layoutAttr)) + composed.emplace(layoutAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + if (!composed || cutegen::has_error(*composed)) + return {nullptr}; + return getAlgebraAttr(context, + std::move(*composed)); +} + +bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, uint32_t bits, + uint32_t base, int32_t shift) { + auto swizzle = cutegen::swizzle::get(bits, base, shift); + if (!swizzle) + return {nullptr}; + return getAlgebraAttr(context, *swizzle); +} + +bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +// Compiler attributes + +bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::BitLayoutAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + layoutValue)); +} +MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayout())); +} + +bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrc, + MlirAttribute layoutDst, + MlirAttribute layoutRef) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; + if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; + if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; + if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) + return {nullptr}; + return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, + layoutDstValue, layoutRefValue)); +} +MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutRef())); +} + +bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute +mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, + MlirAttribute frgSrc, MlirAttribute frgDst) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; + if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; + if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) + return {nullptr}; + mlir::Attribute frgSrcValue; + if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) + return {nullptr}; + mlir::Attribute frgDstValue; + if (!unwrapAs(ctx, frgDst, false, frgDstValue)) + return {nullptr}; + return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, + layoutDstTVValue, frgSrcValue, + frgDstValue)); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSrc())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgDst())); +} + +bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, + frgBValue, frgCValue)); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, + shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, + frgSFAValue, frgBValue, frgSFBValue, frgCValue)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, + frgEValue)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, + shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, + frgSFBValue, frgCValue, frgEValue)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +// Types + +bool mlirTypeIsACuteType(MlirType type) { + mlir::Type cppType = unwrap(type); + return cppType && + cppType.getDialect().getNamespace() == + mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); +} + +bool mlirTypeIsACuteArithTupleIterator(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, + MlirType arithTuple) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); +} +MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} + +bool mlirTypeIsACuteComposedLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ComposedLayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteConstrainedInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); +} +int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibility(); +} +uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow2(); +} + +bool mlirTypeIsACuteCoordTensor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::CoordTensorType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + arithTupleValue, layoutValue)); +} +MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} +MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteCoord(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::CoordAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::CoordType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); +} +uint32_t mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow_2(); +} + +bool mlirTypeIsACuteIntTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::IntTupleType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::LayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteMemRef(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::PtrType ptrValue; + if (!unwrapAs(ctx, ptr, false, ptrValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::MemRefType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + ptrValue, layoutValue)); +} +MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPtr())); +} +MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACutePtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, + uint32_t addressSpace, uint64_t alignment, + MlirAttribute swizzle, MlirAttribute bitlayout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, true, valueTypeValue)) + return {nullptr}; + auto addressSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); + if (!addressSpaceValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; + if (!unwrapAs(ctx, swizzle, true, swizzleValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; + if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) + return {nullptr}; + return wrap(cute::PtrType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valueTypeValue, *addressSpaceValue, alignment, swizzleValue, + bitlayoutValue)); +} +MlirType mlirCutePtrTypeGetValueType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValueType())); +} +uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAddressSpace()); +} +uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { + return llvm::cast(unwrap(type)).getAlignment(); +} +MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getSwizzle())); +} +MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBitlayout())); +} + +bool mlirTypeIsACuteShape(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ShapeType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSparseElem(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, + MlirType physicalType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type physicalTypeValue; + if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) + return {nullptr}; + return wrap(cute::SparseElemType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + numLogical, physicalTypeValue)); +} +int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { + return llvm::cast(unwrap(type)).getNumLogical(); +} +MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPhysicalType())); +} + +bool mlirTypeIsACuteStride(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::StrideAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::StrideType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSwizzle(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::SwizzleType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTile(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::TileAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::TileType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTiledCopy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { + return mlirCuteTiledCopyTypeGetCopyAtom(type); +} +MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledCopyV2(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledMma(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, + MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type mmaAtomValue; + if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; + if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; + if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) + return {nullptr}; + return wrap(cute::TiledMmaType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); +} +MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMmaAtom())); +} +MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { + return mlirCuteTiledMmaTypeGetMmaAtom(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAtomLayout_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPermutation_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetPermutationMNK(type); +} + +bool mlirTypeIsACuteTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, + MlirType const *types) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + llvm::SmallVector typesValue; + for (intptr_t i = 0; i < numTypes; ++i) { + mlir::Type element; + if (!unwrapAs(ctx, types[i], false, element)) + return {nullptr}; + typesValue.push_back(element); + } + return wrap(cute::TupleType::get(ctx, typesValue)); +} +intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTypes().size()); +} +MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { + return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); +} diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h new file mode 100644 index 00000000..002f4885 --- /dev/null +++ b/mlir/dialects/cute_ir/capi.h @@ -0,0 +1,492 @@ +//===- capi.h - C API for the cute dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_IR_CAPI_H +#define MLIR_DIALECTS_CUTE_IR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); + +// Attributes + +// Algebra attributes are built bottom up. A leaf is a static integer, a +// dynamic integer of `width` bits (1, 8, 16, 32 or 64) known to be a multiple +// of `divisibility`, a scaled basis (strides) or an underscore (coordinates +// and tiles); a tuple groups attributes of the same kind. A basis `modes` is +// its mode path, outermost first, which CuTe prints innermost first: +// `1@1@0` is modes {0, 1}. Constructors return null for invalid parameters or +// handles of another context. GetValue returns canonical algebra text owned +// by the attribute's context; callers must not free it. Getters require an +// attribute accepted by the corresponding IsA predicate. + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteIntTupleAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteIntTupleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCoordAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrBasisGet( + MlirContext context, int64_t numerator, int64_t denominator, + intptr_t numModes, int32_t const *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrLayoutGet(MlirContext context, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); + +// `inner` is a swizzle or a layout attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutAttrGet(MlirContext context, MlirAttribute inner, + MlirAttribute offset, MlirAttribute outer); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, + uint32_t bits, + uint32_t base, + int32_t shift); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteSwizzleAttrGetValue(MlirAttribute attr); + +// Compiler attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); + +// `#cute.copy_atom`: Thread and value layouts of a copy atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, + MlirAttribute layoutDst, MlirAttribute layoutRef); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); + +// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy +// atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, + MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); + +// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.reduction_op`: Op for cute reduce operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); + +// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); + +// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata +// operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); + +// Types + +/// Returns true when `type` belongs to the CuTe dialect. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); + +// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutTypeGetAttr(MlirType type); + +// `!cute.i`: Integer of any width with known divisibility and power of two. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, + uint32_t width, + bool isPow2); +MLIR_CAPI_EXPORTED int64_t +mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); + +// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, + MlirType arithTuple, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); + +// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of +// coordinates. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); + +// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod +// operations. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( + MlirContext context, uint32_t width, bool isPow2); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); + +// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); + +// `!cute.layout`: CuTe layout: a shape/stride pair. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); + +// `!cute.memref`: Tensor view: a pointer with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, + MlirType ptr, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); + +// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( + MlirContext context, MlirType valueType, uint32_t addressSpace, + uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); + +// `!cute.shape`: Scalar integer or recursive tuple of shape extents. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); + +// `!cute.sparse_elem`: Logical sparse element stored in a physical type. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, + int numLogical, + MlirType physicalType); +MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteSparseElemTypeGetPhysicalType(MlirType type); + +// `!cute.stride`: Scalar or recursive tuple of stride elements. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); + +// `!cute.swizzle`: CuTe swizzle: S. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); + +// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); + +// `!cute.tiled_copy`: Copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, + MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); +// Former name of mlirCuteTiledCopyTypeGetCopyAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); + +// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); + +// `!cute.tiled_mma`: MMA atom tiled over MNK. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( + MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetMmaAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); + +// `!cute.tuple`: Non-empty tuple of types. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, + intptr_t numTypes, + MlirType const *types); +MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, + intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUTE_IR_CAPI_H diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig new file mode 100644 index 00000000..b277d061 --- /dev/null +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -0,0 +1,2844 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const CuteType = opaque { + const M = mlir.Methods(CuteType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; +}; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute"); +} + +/// `cute.kernel`: marks a function as a CuTe kernel. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute.kernel", .unit(ctx)); +} + +/// The address space of a pointer or memref. +pub const MemorySpace = AddressSpace; + +pub fn pointerType( + ctx: *mlir.Context, + space: MemorySpace, + alignment: u64, + opts: struct { + element_type: ?*const mlir.Type = null, + }, +) !*const mlir.Type { + return (try PtrType.get(ctx, .{ + .valueType = opts.element_type, + .addressSpace = space, + .alignment = alignment, + })).type_(); +} + +pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisible_by: u64) !*const mlir.Type { + return (try ConstrainedIntType.get(ctx, .{ + .divisibility = std.math.cast(i64, divisible_by) orelse return error.InvalidMlir, + .width = switch (width) { + .i32 => 32, + .i64 => 64, + }, + .isPow2 = false, + })).type_(); +} + +/// Algebra attributes are built bottom up from leaves and tuples of the same +/// kind. A dynamic leaf has `width` bits (1, 8, 16, 32 or 64) and is known to +/// be a multiple of `divisibility`. Getter strings are owned by the MLIR +/// context and remain valid until that context is destroyed. +pub const IntTupleAttr = opaque { + const M = mlir.Methods(IntTupleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const IntTupleAttr) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const IntTupleAttr) []const u8 { + return mlir.string(c.mlirCuteIntTupleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const IntTupleAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const CoordAttr = opaque { + const M = mlir.Methods(CoordAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const CoordAttr) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const CoordAttr) []const u8 { + return mlir.string(c.mlirCuteCoordAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const CoordAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ShapeAttr = opaque { + const M = mlir.Methods(ShapeAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const ShapeAttr) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const ShapeAttr) []const u8 { + return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); + } + + pub fn rank(self: *const ShapeAttr) usize { + return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); + } + + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const StrideAttr = opaque { + const M = mlir.Methods(StrideAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first; + /// CuTe prints it innermost first, so `1@1@0` is modes `{ 0, 1 }`. + pub fn getBasis(ctx: *mlir.Context, numerator: i64, denominator: i64, modes: []const i32) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrBasisGet(ctx.ptr(), numerator, denominator, @intCast(modes.len), modes.ptr).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const StrideAttr) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const StrideAttr) []const u8 { + return mlir.string(c.mlirCuteStrideAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const StrideAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const LayoutAttr = opaque { + const M = mlir.Methods(LayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: *const ShapeAttr, + stride: *const StrideAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LayoutAttr { + const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const LayoutAttr) []const u8 { + return mlir.string(c.mlirCuteLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const LayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const TileAttr = opaque { + const M = mlir.Methods(TileAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getLayout(ctx: *mlir.Context, layout: *const LayoutAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrLayoutGet(ctx.ptr(), layout.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const TileAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const TileAttr) []const u8 { + return mlir.string(c.mlirCuteTileAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const TileAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ComposedLayoutAttr = opaque { + const M = mlir.Methods(ComposedLayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + /// `inner ∘ offset ∘ outer`. + pub const InitArgs = struct { + inner: union(enum) { + swizzle: *const SwizzleAttr, + layout: *const LayoutAttr, + }, + offset: *const IntTupleAttr, + outer: *const LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const ComposedLayoutAttr { + const inner = switch (args.inner) { + inline else => |attr| attr.ptr(), + }; + const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), inner, args.offset.ptr(), args.outer.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const ComposedLayoutAttr) []const u8 { + return mlir.string(c.mlirCuteComposedLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const ComposedLayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SwizzleAttr = opaque { + const M = mlir.Methods(SwizzleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + /// `S`. + pub const InitArgs = struct { + bits: u32, + base: u32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const SwizzleAttr { + const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), args.bits, args.base, args.shift); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const SwizzleAttr) []const u8 { + return mlir.string(c.mlirCuteSwizzleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const SwizzleAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Address spaces for CuTe memrefs and pointers. +pub const AddressSpace = enum(u32) { + generic = 0, + gmem = 1, + smem = 3, + rmem = 5, + tmem = 6, + dsmem = 7, + cmem = 4, +}; + +/// Copy operand ID. +pub const CopyOperand = enum(u32) { + S = 0, + D = 1, +}; + +/// Major mode for MMA operations. +pub const MajorMode = enum(u32) { + k = 0, + mn = 1, +}; + +/// MMA operand ID. +pub const MmaOperand = enum(u32) { + A = 0, + B = 1, + C = 2, + D = 3, + SFA = 4, + SFB = 5, + E = 6, +}; + +/// Op for cute reduce operations. +pub const ReductionOp = enum(u32) { + ADD = 0, + MUL = 1, + MIN = 2, + MAX = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. +pub const ArithTupleIteratorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "arith_tuple_iter"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn arithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteArithTupleIteratorTypeGetArithTuple(self.ptr()).ptr.?); + } +}; + +/// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "composed_layout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.i`: Integer of any width with known divisibility and power of two. +pub const ConstrainedIntType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteConstrainedInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisibility: i64, + width: u32, + isPow2: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteConstrainedIntTypeGet(ctx.ptr(), args.divisibility, args.width, args.isPow2); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn divisibility(self: *const Self) i64 { + return c.mlirCuteConstrainedIntTypeGetDivisibility(self.ptr()); + } + pub fn width(self: *const Self) u32 { + return c.mlirCuteConstrainedIntTypeGetWidth(self.ptr()); + } + pub fn isPow2(self: *const Self) bool { + return c.mlirCuteConstrainedIntTypeGetIsPow2(self.ptr()); + } +}; + +/// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "coord_tensor"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + layout: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn arithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetArithTuple(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of coordinates. +pub const CoordType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "coord"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCoordTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod operations. +pub const FastDivmodDivisorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "fast_divmod_divisor"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + width: u32, + isPow2: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn width(self: *const Self) u32 { + return c.mlirCuteFastDivmodDivisorTypeGetWidth(self.ptr()); + } + pub fn isPow2(self: *const Self) bool { + return c.mlirCuteFastDivmodDivisorTypeGetIsPow2(self.ptr()); + } +}; + +/// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "int_tuple"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteIntTupleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.layout`: CuTe layout: a shape/stride pair. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "layout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.memref`: Tensor view: a pointer with a layout. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "memref"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + ptr: *const mlir.Type, + layout: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn ptr_(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetPtr(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. +pub const PtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACutePtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "ptr"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: ?*const mlir.Type = null, + addressSpace: AddressSpace, + alignment: u64, + swizzle: ?*const mlir.Attribute = null, + bitlayout: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valueType(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); + } + pub fn addressSpace(self: *const Self) AddressSpace { + return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); + } + pub fn alignment(self: *const Self) u64 { + return c.mlirCutePtrTypeGetAlignment(self.ptr()); + } + pub fn swizzle(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); + } + pub fn bitlayout(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetBitlayout(self.ptr()).ptr); + } +}; + +/// `!cute.shape`: Scalar integer or recursive tuple of shape extents. +pub const ShapeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "shape"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteShapeTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.sparse_elem`: Logical sparse element stored in a physical type. +pub const SparseElemType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSparseElem; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_elem"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + numLogical: c_int, + physicalType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numLogical(self: *const Self) c_int { + return c.mlirCuteSparseElemTypeGetNumLogical(self.ptr()); + } + pub fn physicalType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteSparseElemTypeGetPhysicalType(self.ptr()).ptr.?); + } +}; + +/// `!cute.stride`: Scalar or recursive tuple of stride elements. +pub const StrideType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "stride"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteStrideTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.swizzle`: CuTe swizzle: S. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "swizzle"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSwizzleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tile"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.tiled_copy`: Copy atom tiled over threads and values. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyTypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn copyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyTypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn tilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetTilerMn(self.ptr()).ptr.?); + } + pub const atom = copyAtom; +}; + +/// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +pub const TiledCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy_v2"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyV2TypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn copyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn tilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetTilerMn(self.ptr()).ptr.?); + } +}; + +/// `!cute.tiled_mma`: MMA atom tiled over MNK. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + mmaAtom: *const mlir.Type, + atomLayoutMNK: *const mlir.Attribute, + permutationMNK: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledMmaTypeGet(ctx.ptr(), args.mmaAtom.ptr(), args.atomLayoutMNK.ptr(), if (args.permutationMNK) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn mmaAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledMmaTypeGetMmaAtom(self.ptr()).ptr.?); + } + pub fn atomLayoutMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetAtomLayoutMNK(self.ptr()).ptr.?); + } + pub fn permutationMNK(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetPermutationMNK(self.ptr()).ptr); + } + pub const atom = mmaAtom; + pub const getAtomLayoutMnk = atomLayoutMNK; + pub const getPermutationMnk = permutationMNK; +}; + +/// `!cute.tuple`: Non-empty tuple of types. +pub const TupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tuple"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + types: []const *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTupleTypeGet(ctx.ptr(), @intCast(args.types.len), @ptrCast(args.types.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numTypes(self: *const Self) usize { + return @intCast(c.mlirCuteTupleTypeGetNumTypes(self.ptr())); + } + pub fn @"type"(self: *const Self, pos: usize) *const mlir.Type { + return @ptrCast(c.mlirCuteTupleTypeGetType(self.ptr(), @intCast(pos)).ptr.?); + } +}; + +/// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +pub const BitLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteBitLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bitlayout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + layout: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteBitLayoutAttrGet(ctx.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteBitLayoutAttrGetLayout(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom`: Thread and value layouts of a copy atom. +pub const CopyAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrc: *const mlir.Attribute, + layoutDst: *const mlir.Attribute, + layoutRef: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomAttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrc.ptr(), args.layoutDst.ptr(), args.layoutRef.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutSrc(self.ptr()).ptr.?); + } + pub fn layoutDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutDst(self.ptr()).ptr.?); + } + pub fn layoutRef(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy atom. +pub const CopyAtomV2Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtomV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_atom_v2"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrcTV: *const mlir.Attribute, + layoutDstTV: *const mlir.Attribute, + frgSrc: *const mlir.Attribute, + frgDst: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomV2AttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrcTV.ptr(), args.layoutDstTV.ptr(), args.frgSrc.ptr(), args.frgDst.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutSrcTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutSrcTV(self.ptr()).ptr.?); + } + pub fn layoutDstTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutDstTV(self.ptr()).ptr.?); + } + pub fn frgSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgSrc(self.ptr()).ptr.?); + } + pub fn frgDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgDst(self.ptr()).ptr.?); + } +}; + +/// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +pub const MmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +pub const MxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mx_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.reduction_op`: Op for cute reduce operations. +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionOp { + return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +pub const SparseMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn layoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn frgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +/// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata operands. +pub const SparseMxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_mx_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn layoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn frgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.add_offset", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.append_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.append_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. +pub fn apply_swizzle( + ctx: *mlir.Context, + ptr: *const mlir.Value, + swizzled_ptr_type: *const mlir.Type, + opts: struct { + swizzle: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); + return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{swizzled_ptr_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. +pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.assume", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.blocked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn blocked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.blocked_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.ceil_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ceil_div", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. +pub fn coalesce( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.coalesce", .{ + .operands = .{ .variadic = &.{ + &.{input}, + if (opts.target_profile) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.complement`. Result types are explicit; attributes use mlir.Attribute. +pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.complement", .{ + .operands = .{ .flat = &.{ input, cotarget } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.composed_get_inner`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_outer`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composition`. Result types are explicit; attributes use mlir.Attribute. +pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composition", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + src, + dst, + if (opts.pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.copy.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_atom_call( + ctx: *mlir.Context, + atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + dst, + if (opts.pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. +pub fn cosize( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.cosize", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, index_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .results = .{ .flat = &.{index_type} }, + .location = location, + }); +} + +/// `cute.deref_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith_tuple_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_arith_tuple_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{arith_tuple_type} }, + .location = location, + }); +} + +/// `cute.deref_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_desc_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{value__type} }, + .location = location, + }); +} + +/// `cute.derefine`. Result types are explicit; attributes use mlir.Attribute. +pub fn derefine(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.derefine", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. +pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, coord: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.dice", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "coord", coord), + }, + .location = location, + }); +} + +/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.compute`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_compute(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, remainder_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.compute", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{ quotient_type, remainder_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.create_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_create_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.create_divisor", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_divide(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.divide", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{quotient_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_aux`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_aux(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier_type: *const mlir.Type, shift1_type: *const mlir.Type, shift2_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_aux", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{ multiplier_type, shift1_type, shift2_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_divisor(ctx: *mlir.Context, fast_divmod_divisor: *const mlir.Value, divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_divisor", .{ + .operands = .{ .flat = &.{fast_divmod_divisor} }, + .results = .{ .flat = &.{divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.make_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_make_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier: *const mlir.Value, sh1: *const mlir.Value, sh2: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.make_divisor", .{ + .operands = .{ .flat = &.{ divisor, multiplier, sh1, sh2 } }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.filter`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.filter", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter_zeros( + ctx: *mlir.Context, + input: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.filter_zeros", .{ + .operands = .{ .variadic = &.{ + &.{input}, + if (opts.target_profile) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.gemm`. Result types are explicit; attributes use mlir.Attribute. +pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. +pub fn get( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.get", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_flat_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_hier_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_hier_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_hier_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_integral_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_integral_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_integral_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_iter(ctx: *mlir.Context, source: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_iter", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute.get_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layout(ctx: *mlir.Context, input: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layout", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ + .operands = .{ .flat = &.{tile} }, + .results = .{ .variadic = &.{ + layouts_types, + } }, + .location = location, + }); +} + +/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results__types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .variadic = &.{ + results__types, + } }, + .location = location, + }); +} + +/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_scalars( + ctx: *mlir.Context, + cute_value: *const mlir.Value, + scalars_types: []const *const mlir.Type, + opts: struct { + only_dynamic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); + return mlir.Operation.make(ctx, "cute.get_scalars", .{ + .operands = .{ .flat = &.{cute_value} }, + .results = .{ .variadic = &.{ + scalars_types, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_shape", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_stride", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. +pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.group_modes", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .location = location, + }); +} + +/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. +pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, coord_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.idx2crd", .{ + .operands = .{ .flat = &.{ index, shape } }, + .results = .{ .flat = &.{coord_type} }, + .location = location, + }); +} + +/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.increment_coord", .{ + .operands = .{ .flat = &.{ coord, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.inttoptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.kernel_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn kernel_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, kernel_name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.kernel_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = &.{ + .named(ctx, "kernel_name", kernel_name), + }, + .location = location, + }); +} + +/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.left_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.load_scaled_index`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_scaled_index(ctx: *mlir.Context, ptr_a: *const mlir.Value, ptr_b: *const mlir.Value, index: *const mlir.Value, stride: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.load_scaled_index", .{ + .operands = .{ .flat = &.{ ptr_a, ptr_b, index, stride } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_partition(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, index: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.local_partition", .{ + .operands = .{ .flat = &.{ input, tiler, index } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_tile( + ctx: *mlir.Context, + input: *const mlir.Value, + tile: *const mlir.Value, + coord: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + proj: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); + return mlir.Operation.make(ctx, "cute.local_tile", .{ + .operands = .{ .flat = &.{ input, tile, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_arith_tuple_iter( + ctx: *mlir.Context, + iter_type: *const mlir.Type, + opts: struct { + value_: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ + .operands = .{ .variadic = &.{ + if (opts.value_) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_atom`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_atom(ctx: *mlir.Context, values: []const *const mlir.Value, atom_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_atom", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .results = .{ .flat = &.{atom_type} }, + .location = location, + }); +} + +/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_coord(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_coord", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_desc_iter(ctx: *mlir.Context, value_: *const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_desc_iter", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_fragment_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_identity_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_tensor", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_int_tuple(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout( + ctx: *mlir.Context, + layout_type: *const mlir.Type, + opts: struct { + shape: ?*const mlir.Value = null, + stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout", .{ + .operands = .{ .variadic = &.{ + if (opts.shape) |value| &.{value} else &.{}, + if (opts.stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_shape(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_shape", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_sparse_elem`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_sparse_elem", .{ + .operands = .{ .flat = &.{physical_storage} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_stride(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_stride", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tile(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tile", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy_v2(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy_v2", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_mma(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_mma", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tuple", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_view( + ctx: *mlir.Context, + iter: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_view", .{ + .operands = .{ .variadic = &.{ + &.{iter}, + if (opts.layout) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloc_smem", .{ + .results = .{ .flat = &.{memref_type} }, + .location = location, + }); +} + +/// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloca", .{ + .operands = .{ .variadic = &.{ + if (opts.layout) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.load", .{ + .operands = .{ .flat = &.{ src, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load_vec( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + pass_thru: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.mask) |value| &.{value} else &.{}, + if (opts.pass_thru) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const mlir.Value, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.store", .{ + .operands = .{ .flat = &.{ dst, coord, value_ } }, + .location = location, + }); +} + +/// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store_vec( + ctx: *mlir.Context, + value_: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ + .operands = .{ .variadic = &.{ + &.{value_}, + &.{dst}, + if (opts.mask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.mma_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{prefetch_atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.prefetch_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn print( + ctx: *mlir.Context, + values: []const *const mlir.Value, + opts: struct { + fmt: ?*const mlir.Attribute = null, + operand_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); + if (opts.operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); + return mlir.Operation.make(ctx, "cute.print", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.print_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_im2col(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_tiled", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_view( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + coord: ?*const mlir.Value = null, + verbose: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "cute.print_view", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.coord) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.ptr.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{value__type} }, + .location = location, + }); +} + +/// `cute.ptr.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_store(ctx: *mlir.Context, value_: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.store", .{ + .operands = .{ .flat = &.{ value_, ptr } }, + .location = location, + }); +} + +/// `cute.ptrtoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptrtoint(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptrtoint", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn raked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.raked_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.random_delay`. Result types are explicit; attributes use mlir.Attribute. +pub fn random_delay(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.random_delay", .{ + .location = location, + }); +} + +/// `cute.recast_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .location = location, + }); +} + +/// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + shape: *const mlir.Value, + init_val: *const mlir.Value, + output_type: *const mlir.Type, + reduction_op: *const mlir.Attribute, + opts: struct { + predication: ?*const mlir.Value = null, + reduced_shape_type: ?*const mlir.Type = null, + reduction_profile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); + if (opts.reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); + return mlir.Operation.make(ctx, "cute.reduce", .{ + .operands = .{ .variadic = &.{ + &.{input}, + &.{shape}, + &.{init_val}, + if (opts.predication) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{ + &.{output_type}, + if (opts.reduced_shape_type) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.right_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. +pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.select", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.shape_div", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn size( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.size", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. +pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.slice", .{ + .operands = .{ .flat = &.{ input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn smem_partition_size( + ctx: *mlir.Context, + size_type: *const mlir.Type, + partition_id: *const mlir.Attribute, + opts: struct { + cumulative: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "partition_id", partition_id)); + if (opts.cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); + return mlir.Operation.make(ctx, "cute.smem_partition_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn stencil_divide( + ctx: *mlir.Context, + input: *const mlir.Value, + stencil: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + padding_upper: ?*const mlir.Value = null, + padding_lower: ?*const mlir.Value = null, + traversal_stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.stencil_divide", .{ + .operands = .{ .variadic = &.{ + &.{input}, + &.{stencil}, + if (opts.padding_upper) |value| &.{value} else &.{}, + if (opts.padding_lower) |value| &.{value} else &.{}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.symbolic`. Result types are explicit; attributes use mlir.Attribute. +pub fn symbolic(ctx: *mlir.Context, symbolic_type: *const mlir.Type, name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.symbolic", .{ + .results = .{ .flat = &.{symbolic_type} }, + .attributes = &.{ + .named(ctx, "name", name), + }, + .location = location, + }); +} + +/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_D`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_D(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_D", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_S`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_S(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_S", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.retile`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mod`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.unpack_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn unpack_tuple(ctx: *mlir.Context, tuple: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.unpack_tuple", .{ + .operands = .{ .flat = &.{tuple} }, + .results = .{ .variadic = &.{ + result_types, + } }, + .location = location, + }); +} + +/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} diff --git a/mlir/dialects/cute_ir/test.zig b/mlir/dialects/cute_ir/test.zig new file mode 100644 index 00000000..c6f44763 --- /dev/null +++ b/mlir/dialects/cute_ir/test.zig @@ -0,0 +1,341 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const ArithTupleIteratorType = cute_ir.ArithTupleIteratorType; +const ComposedLayoutType = cute_ir.ComposedLayoutType; +const ConstrainedIntType = cute_ir.ConstrainedIntType; +const CoordTensorType = cute_ir.CoordTensorType; +const CoordType = cute_ir.CoordType; +const FastDivmodDivisorType = cute_ir.FastDivmodDivisorType; +const IntTupleType = cute_ir.IntTupleType; +const LayoutType = cute_ir.LayoutType; +const MemRefType = cute_ir.MemRefType; +const PtrType = cute_ir.PtrType; +const ShapeType = cute_ir.ShapeType; +const SparseElemType = cute_ir.SparseElemType; +const StrideType = cute_ir.StrideType; +const SwizzleType = cute_ir.SwizzleType; +const TileType = cute_ir.TileType; +const TiledCopyType = cute_ir.TiledCopyType; +const TiledCopyV2Type = cute_ir.TiledCopyV2Type; +const TiledMmaType = cute_ir.TiledMmaType; +const TupleType = cute_ir.TupleType; +const BitLayoutAttr = cute_ir.BitLayoutAttr; +const CopyAtomAttr = cute_ir.CopyAtomAttr; +const CopyAtomV2Attr = cute_ir.CopyAtomV2Attr; +const MmaAtomAttr = cute_ir.MmaAtomAttr; +const MxMmaAtomAttr = cute_ir.MxMmaAtomAttr; +const SparseMmaAtomAttr = cute_ir.SparseMmaAtomAttr; +const SparseMxMmaAtomAttr = cute_ir.SparseMxMmaAtomAttr; + +const operation_names: []const []const u8 = &.{ + "cute.add_offset", + "cute.append_to_rank", + "cute.apply_swizzle", + "cute.assume", + "cute.blocked_product", + "cute.ceil_div", + "cute.coalesce", + "cute.complement", + "cute.composed_get_inner", + "cute.composed_get_offset", + "cute.composed_get_outer", + "cute.composition", + "cute.copy", + "cute.copy.make_fragment", + "cute.copy_atom_call", + "cute.cosize", + "cute.crd2idx", + "cute.deref_arith_tuple_iter", + "cute.deref_desc_iter", + "cute.derefine", + "cute.dice", + "cute.elem_less", + "cute.equal", + "cute.fast_divmod.compute", + "cute.fast_divmod.create_divisor", + "cute.fast_divmod.divide", + "cute.fast_divmod.get_aux", + "cute.fast_divmod.get_divisor", + "cute.fast_divmod.make_divisor", + "cute.filter", + "cute.filter_zeros", + "cute.flat_divide", + "cute.flat_product", + "cute.gemm", + "cute.get", + "cute.get_flat_coord", + "cute.get_hier_coord", + "cute.get_integral_coord", + "cute.get_iter", + "cute.get_layout", + "cute.get_layouts_from_tile", + "cute.get_leaves", + "cute.get_scalars", + "cute.get_shape", + "cute.get_stride", + "cute.group_modes", + "cute.idx2crd", + "cute.increment_coord", + "cute.inttoptr", + "cute.kernel_smem_size", + "cute.left_inverse", + "cute.load_scaled_index", + "cute.local_partition", + "cute.local_tile", + "cute.logical_divide", + "cute.logical_product", + "cute.make_arith_tuple_iter", + "cute.make_atom", + "cute.make_composed_layout", + "cute.make_coord", + "cute.make_desc_iter", + "cute.make_fragment_like", + "cute.make_identity_layout", + "cute.make_identity_tensor", + "cute.make_int_tuple", + "cute.make_layout", + "cute.make_layout_like", + "cute.make_ordered_layout", + "cute.make_shape", + "cute.make_sparse_elem", + "cute.make_stride", + "cute.make_tile", + "cute.make_tiled_copy", + "cute.make_tiled_copy_v2", + "cute.make_tiled_mma", + "cute.make_tuple", + "cute.make_view", + "cute.memref.alloc_smem", + "cute.memref.alloca", + "cute.memref.load", + "cute.memref.load_vec", + "cute.memref.store", + "cute.memref.store_vec", + "cute.mma.make_fragment", + "cute.mma_atom_call", + "cute.prefetch", + "cute.prefetch_atom_call", + "cute.prepend_to_rank", + "cute.print", + "cute.print_tma_desc_im2col", + "cute.print_tma_desc_tiled", + "cute.print_view", + "cute.ptr.load", + "cute.ptr.store", + "cute.ptrtoint", + "cute.raked_product", + "cute.random_delay", + "cute.recast_iter", + "cute.recast_layout", + "cute.reduce", + "cute.right_inverse", + "cute.select", + "cute.shape_div", + "cute.size", + "cute.slice", + "cute.smem_partition_size", + "cute.static", + "cute.stencil_divide", + "cute.symbolic", + "cute.tile_to_shape", + "cute.tiled.copy.partition_D", + "cute.tiled.copy.partition_S", + "cute.tiled.copy.retile", + "cute.tiled.mma.partition", + "cute.tiled.mma.partition_shape", + "cute.tiled_divide", + "cute.tiled_product", + "cute.to_int_tuple", + "cute.tuple.product", + "cute.tuple.product_each", + "cute.tuple_add", + "cute.tuple_div", + "cute.tuple_mod", + "cute.tuple_mul", + "cute.tuple_sub", + "cute.unpack_tuple", + "cute.zipped_divide", + "cute.zipped_product", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_ir); + inline for (comptime std.meta.declarations(cute_ir)) |decl| { + const T = @field(cute_ir, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "kernel is a unit attribute of the cute dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_ir.kernel(ctx)})}); + try std.testing.expectEqualStrings("{cute.kernel}", w.buffered()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, + .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, + .{ ConstrainedIntType, "!cute.i16" }, + .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, + .{ CoordType, "!cute.coord<\"1\">" }, + .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, + .{ IntTupleType, "!cute.int_tuple<\"1\">" }, + .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, + .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, + .{ PtrType, "!cute.ptr, S<2,4,3>>" }, + .{ ShapeType, "!cute.shape<\"(2,3)\">" }, + .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, + .{ StrideType, "!cute.stride<\"1\">" }, + .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, + .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, + .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, + .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, + .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, + .{ TupleType, "!cute.tuple" }, + .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, + .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, + .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_ir}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } +} + +test "algebra attributes build from their leaves and tuples" { + const ctx = try testContext(); + defer ctx.deinit(); + + const int_tuple = try cute_ir.IntTupleAttr.getTuple(ctx, &.{ + try .getStatic(ctx, 0), + try .getDynamic(ctx, 32, 1), + try .getDynamic(ctx, 32, 16), + try .getTuple(ctx, &.{ try .getStatic(ctx, 4), try .getDynamic(ctx, 64, 1) }), + }); + try expectType(ctx, "!cute.int_tuple<\"(0,?,?{div=16},(4,?{i64}))\">", (try IntTupleType.get(ctx, .{ .attr = int_tuple.attribute() })).type_()); + try expectType(ctx, "!cute.int_tuple<\"()\">", (try IntTupleType.get(ctx, .{ .attr = (try cute_ir.IntTupleAttr.getTuple(ctx, &.{})).attribute() })).type_()); + + const coord = try cute_ir.CoordAttr.getTuple(ctx, &.{ + try .getUnderscore(ctx), + try .getStatic(ctx, 1), + try .getTuple(ctx, &.{ try .getDynamic(ctx, 32, 1), try .getUnderscore(ctx) }), + }); + try expectType(ctx, "!cute.coord<\"(_,1,(?,_))\">", (try CoordType.get(ctx, .{ .attr = coord.attribute() })).type_()); + + const shape = try cute_ir.ShapeAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getStatic(ctx, 512), try .getStatic(ctx, 1) }), + try .getStatic(ctx, 1), + }); + const stride = try cute_ir.StrideAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getBasis(ctx, 1, 2, &.{0}), try .getStatic(ctx, 0) }), + try .getBasis(ctx, 3, 1, &.{ 0, 1 }), + }); + const layout = try cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = stride }); + try expectType(ctx, "!cute.layout<\"((512,1),1):((1/2@0,0),3@1@0)\">", (try LayoutType.get(ctx, .{ .attr = layout.attribute() })).type_()); + try std.testing.expectError(error.InvalidMlir, cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = try .getStatic(ctx, 1) })); + + const row = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getDynamic(ctx, 32, 8) }), + }); + try expectType(ctx, "!cute.layout<\"(8,64):(64,?{div=8})\">", (try LayoutType.get(ctx, .{ .attr = row.attribute() })).type_()); + + const tile = try cute_ir.TileAttr.getTuple(ctx, &.{ try .getLayout(ctx, row), try .getUnderscore(ctx) }); + try expectType(ctx, "!cute.tile<\"[(8,64):(64,?{div=8});_]\">", (try TileType.get(ctx, .{ .attr = tile.attribute() })).type_()); + + const swizzle = try cute_ir.SwizzleAttr.get(ctx, .{ .bits = 3, .base = 4, .shift = 3 }); + try expectType(ctx, "!cute.swizzle<\"S<3,4,3>\">", (try SwizzleType.get(ctx, .{ .attr = swizzle.attribute() })).type_()); + + const outer = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getStatic(ctx, 1) }), + }); + const composed = try cute_ir.ComposedLayoutAttr.get(ctx, .{ .inner = .{ .swizzle = swizzle }, .offset = try .getStatic(ctx, 0), .outer = outer }); + try expectType(ctx, "!cute.composed_layout<\"S<3,4,3> o 0 o (8,64):(64,1)\">", (try ComposedLayoutType.get(ctx, .{ .attr = composed.attribute() })).type_()); + + try std.testing.expectError(error.InvalidMlir, cute_ir.IntTupleAttr.getDynamic(ctx, 7, 1)); + try std.testing.expectError(error.InvalidMlir, cute_ir.StrideAttr.getBasis(ctx, 1, 1, &.{})); + try std.testing.expectError(error.InvalidMlir, cute_ir.SwizzleAttr.get(ctx, .{ .bits = 5, .base = 4, .shift = 3 })); +} + +fn expectType(ctx: *mlir.Context, expected: []const u8, actual: *const mlir.Type) !void { + const parsed = try mlir.Type.parse(ctx, expected); + if (!parsed.eql(actual)) { + std.debug.print("expected {s}, built {f}\n", .{ expected, actual }); + return error.TestUnexpectedResult; + } +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cute_nvgpu/BUILD.bazel b/mlir/dialects/cute_nvgpu/BUILD.bazel new file mode 100644 index 00000000..07c40ea9 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/BUILD.bazel @@ -0,0 +1,192 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_ir:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cute_nvgpu", + import_name = "mlir/dialects/cute_nvgpu", + main = "cute_nvgpu.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/cute_ir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cute_nvgpu", + "//mlir", + "//mlir/dialects/cute_ir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CuteNVGPUAttrs.td", + "CuteNVGPUDialect.td", + "CuteNVGPUEnums.td", + "CuteNVGPUOps.td", + "CuteNVGPUTypes.td", + ], + includes = ["/"], + deps = [ + "//mlir/dialects/cute_ir:td_files", + "@cute_ir//:cute_ir_dialect_td", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUTypes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteNVGPUEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteNVGPUEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteNVGPUOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteNVGPUOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CuteNVGPUDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = ["CuteNVGPUDialect.cpp"], + deps = [ + ":headers", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AsmParser", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td new file mode 100644 index 00000000..b4230c61 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td @@ -0,0 +1,132 @@ +//===- CuteNVGPUAttrs.td - CuTe NVIDIA GPU attributes ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and probing its attribute parser and +// printer) and its enum bindings; do not edit by hand. +// +// The enum attributes are CuteNVGPUEnums.td; `#cute_nvgpu.major`, over the +// cute dialect's MajorMode, is here so that the enum generator run over +// CuteNVGPUEnums.td emits the nvgpu enums only. The fragment attributes are +// `key = value` lists in a fixed order; a defaulted field is printed only +// when set. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +def CuteNVGPU_MajorModeAttr + : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +class CuteNVGPU_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +// `true` / `false`. +class CuteNVGPU_BoolParam : AttrOrTypeParameter<"bool", ""> { + let parser = "::mlir::cutlass_compiler::cute_nvgpu::parseBool($_parser)"; + let printer = "$_printer << ($_self ? \"true\" : \"false\")"; +} +def CuteNVGPU_Bool : CuteNVGPU_BoolParam; +def CuteNVGPU_FalseByDefault : CuteNVGPU_BoolParam { + let defaultValue = "false"; +} + +def CuteNVGPU_RmemFrgAttr : CuteNVGPU_Attr<"RmemFrg", "rmem_frg"> { + let summary = "Register-memory fragment of an MMA operand"; + let description = [{ + `#cute_nvgpu.rmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$value_type, + "::mlir::cutlass_compiler::cute::MmaOperand":$operand, + CuteNVGPU_FalseByDefault:$derive_elem_type + ); + let assemblyFormat = [{ + `<` `value_type` `=` $value_type `,` `operand` `=` $operand + (`,` `derive_elem_type` `=` $derive_elem_type^)? `>` + }]; +} + +class CuteNVGPU_SmemFrg + : CuteNVGPU_Attr { + let summary = arch # " shared-memory fragment"; + let description = "`#cute_nvgpu." # attrMnemonic # "`"; + let parameters = (ins "::mlir::cutlass_compiler::cute::MajorMode":$major_mode); + let assemblyFormat = "`<` `major` `=` $major_mode `>`"; +} + +def CuteNVGPU_SM90SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM90SmemFrg", "arch.sm90.smem_frg", "SM90">; +def CuteNVGPU_SM100SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100SmemFrg", "arch.sm100.smem_frg", "SM100">; +def CuteNVGPU_SM100CircularSmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100CircularSmemFrg", "arch.sm100.circular_smem_frg", + "SM100 circular">; +def CuteNVGPU_SM107SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM107SmemFrg", "arch.sm107.smem_frg", "SM107">; + +def CuteNVGPU_SM100TmemFrgAttr + : CuteNVGPU_Attr<"SM100TmemFrg", "arch.sm100.tmem_frg"> { + let summary = "SM100 tensor-memory fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$data_type, + "::mlir::Type":$storage_type, + "int":$cta_group, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `data_type` `=` $data_type `,` `storage_type` `=` $storage_type `,` + `cta_group` `=` $cta_group `,` `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemSfFrgAttr + : CuteNVGPU_Attr<"SM100TmemSfFrg", "arch.sm100.tmem_sf_frg"> { + let summary = "SM100 tensor-memory scale-factor fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_sf_frg` + }]; + let parameters = (ins + "::mlir::Type":$sf_type, + "int":$sf_vec_size, + "int":$cta_group, + CuteNVGPU_Bool:$is_sfa, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `sf_type` `=` $sf_type `,` `sf_vec_size` `=` $sf_vec_size `,` + `cta_group` `=` $cta_group `,` `is_sfa` `=` $is_sfa `,` + `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemEFrgAttr + : CuteNVGPU_Attr<"SM100TmemEFrg", "arch.sm100.tmem_e_frg"> { + let summary = "SM100 tensor-memory sparse-metadata fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_e_frg` + }]; + let parameters = (ins "::mlir::Type":$a_type, "::mlir::Type":$e_type); + let assemblyFormat = "`<` `a_type` `=` $a_type `,` `e_type` `=` $e_type `>`"; +} + +def CuteNVGPU_NotImplementedFrgAttr + : CuteNVGPU_Attr<"NotImplementedFrg", "not_implemented_frg"> { + let summary = "Fragment of an atom that has none"; + let description = "`#cute_nvgpu.not_implemented_frg<>`"; + let assemblyFormat = "`<` `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp new file mode 100644 index 00000000..a912714e --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp @@ -0,0 +1,1921 @@ +//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect ---------------------===// +// +// Parsers, printers and verifiers of the cute_nvgpu types that the declarative +// format cannot express. The syntax and the verifier messages are the DSL +// compiler's (see CuteNVGPUTypes.td). + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/AsmParser/AsmParser.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" + +#include +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute_nvgpu; +namespace cute = mlir::cutlass_compiler::cute; + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp.inc" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" + +FailureOr mlir::cutlass_compiler::cute_nvgpu::parseBool(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (kw == "true" || kw == "false") + return kw == "true"; + return p.emitError(loc, "expected `true` or `false`"); +} + +//===----------------------------------------------------------------------===// +// Helpers +//===----------------------------------------------------------------------===// + +namespace { + +using EmitFn = llvm::function_ref; + +template ParseResult field(AsmParser &p, T &out) { + FailureOr v = FieldParser::parse(p); + if (failed(v)) + return failure(); + out = *v; + return success(); +} +ParseResult field(AsmParser &p, Type &out) { return p.parseType(out); } +ParseResult field(AsmParser &p, int &out) { return p.parseInteger(out); } +ParseResult field(AsmParser &p, bool &out) { + FailureOr v = parseBool(p); + if (failed(v)) + return failure(); + out = *v; + return success(); +} + +// `key = value`. +template +ParseResult keyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseKeyword(key) || p.parseEqual() || field(p, out)); +} +// `, key = value`. +template +ParseResult nextKeyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseComma() || keyField(p, key, out)); +} +// An optional ` key = value` group. +template +ParseResult optKeyField(AsmParser &p, StringRef key, T &out) { + if (failed(p.parseOptionalKeyword(key))) + return success(); + return failure(p.parseEqual() || field(p, out)); +} +// `N word`, as in `128 b` or `32 DP`. +ParseResult countOf(AsmParser &p, StringRef word, int &out) { + return failure(p.parseInteger(out) || p.parseKeyword(word)); +} +// `(a, b)`. +template ParseResult pairOf(AsmParser &p, T &a, T &b) { + return failure(p.parseLParen() || field(p, a) || p.parseComma() || + field(p, b) || p.parseRParen()); +} +// `elem_type = (A, B, C)`. +ParseResult elemTypes(AsmParser &p, Type &a, Type &b, Type &c) { + return failure(p.parseKeyword("elem_type") || p.parseEqual() || + p.parseLParen() || p.parseType(a) || p.parseComma() || + p.parseType(b) || p.parseComma() || p.parseType(c) || + p.parseRParen()); +} +void printElemTypes(AsmPrinter &p, Type a, Type b, Type c) { + p << "elem_type = (" << a << ", " << b << ", " << c << ")"; +} +// `xR`. +ParseResult repeat(AsmParser &p, unsigned &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (!kw.consume_front("x")) + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to start with `x`"); + if (kw.getAsInteger(10, out)) + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); + return success(); +} + +template +void printIfNot(AsmPrinter &p, StringRef key, E value, E dflt) { + if (value != dflt) + p << ' ' << key << " = " << stringifyEnum(value); +} +template +void printAttrField(AsmPrinter &p, StringRef key, A attr) { + if (!attr) + return; + p << ", " << key << " = "; + p.printStrippedAttrOrType(attr); +} + +// The width of an element: integers and floats, 64 for index and pointers, +// a sparse element's share of its physical storage. +int elementBits(Type t) { + if (auto s = llvm::dyn_cast(t)) + return s.getNumLogical() + ? elementBits(s.getPhysicalType()) / s.getNumLogical() + : 0; + if (t.isTF32()) + return 32; + if (t.isIntOrFloat()) + return t.getIntOrFloatBitWidth(); + if (llvm::isa(t)) + return 64; + return 0; +} +int orElementBits(int bits, Type t) { return bits ? bits : elementBits(t); } + +// The integers of a flat tuple `(a,b,c)`. +std::optional> flatInts(StringRef text) { + if (!text.consume_front("(") || !text.consume_back(")")) + return std::nullopt; + SmallVector out; + for (StringRef part : llvm::split(text, ',')) { + int64_t v; + if (part.getAsInteger(10, v)) + return std::nullopt; + out.push_back(v); + } + return out; +} + +// `MxNxK`. +ParseResult parseShapeMNK(AsmParser &p, cute::ShapeAttr &out) { + SMLoc loc = p.getCurrentLocation(); + SmallVector dims; + if (p.parseDimensionList(dims, /*allowDynamic=*/false, + /*withTrailingX=*/false)) + return p.emitError(loc, "can not parse shape_MNK"); + auto shape = cutegen::from_string( + "(" + + llvm::join( + llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), + ",") + + ")"); + if (!shape) + return p.emitError(loc, "can not parse shape_MNK"); + out = cute::ShapeAttr::get(p.getContext(), *shape); + return success(); +} +void printShapeMNK(AsmPrinter &p, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + StringRef body(text); + body.consume_front("("); + body.consume_back(")"); + for (char c : body) + p << (c == ',' ? 'x' : c); +} +LogicalResult verifyShapeMNK(EmitFn emitError, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + auto dims = flatInts(text); + if (!dims || dims->size() != 3) + return emitError() << "expects flatten rank-3 tuple of shape, but got " + << text; + return success(); +} + +// The A fragment kind: `ss` (shared memory), `rs` (registers) or `ts` +// (tensor memory). +ParseResult parseFragKind(AsmParser &p, MmaFragKind other, MmaFragKind &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + bool ts = other == MmaFragKind::tmem; + if (p.parseKeyword("frag_kind") || p.parseEqual() || p.parseKeyword(&kw)) + return failure(); + if (kw == "ss") + out = MmaFragKind::smem_desc; + else if (kw == (ts ? "ts" : "rs")) + out = other; + else + return p.emitError(loc, + ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); + return success(); +} +StringRef fragKindText(MmaFragKind k) { + switch (k) { + case MmaFragKind::smem_desc: + return "ss"; + case MmaFragKind::rmem: + return "rs"; + case MmaFragKind::tmem: + return "ts"; + default: + return stringifyEnum(k); + } +} + +// `t` or `n`. +ParseResult parseTranspose(AsmParser &p, UnitAttr &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (kw != "t" && kw != "n") + return p.emitError(loc, "expects `t` or `n`"); + out = kw == "t" ? UnitAttr::get(p.getContext()) : UnitAttr(); + return success(); +} + +bool isPow2AtLeast8(int v) { return v >= 8 && llvm::isPowerOf2_32(v); } + +} // namespace + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + +// sm103/sm107.smem_desc_circular: static `(B,128,N):(128,1,128*B)`, N >= 2. +static LogicalResult verifyCircular(EmitFn emitError, cute::LayoutAttr layout) { + std::string text = cutegen::to_string(layout.getRef()); + StringRef shapeText, strideText; + std::tie(shapeText, strideText) = StringRef(text).split(':'); + if (StringRef(text).contains('?')) + return emitError() << "block layout must be fully static but got " << text; + auto shape = flatInts(shapeText), stride = flatInts(strideText); + if (!shape || shape->size() != 3 || !stride || stride->size() != 3) + return emitError() << "expects block layout to have 3 modes, but got " + << text; + if ((*shape)[1] != 128) + return emitError() << "expects block layout to have 128B 1st mode, but got " + << text; + if ((*stride)[1] != 1 || (*stride)[0] != 128 || + (*stride)[2] != 128 * (*shape)[0]) + return emitError() << "expects mode-1 major compact layout, but got " + << text; + if ((*shape)[2] < 2) + return emitError() + << "Circular SMEM descriptor requires at least 2 128B blocks since " + "MMA may span 2 buffers, but got " + << text; + return success(); +} +LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} +LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} + +Type SmemDescViewType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type desc; + std::string text; + if (p.parseLess() || p.parseType(desc) || p.parseComma()) + return {}; + SMLoc layoutLoc = p.getCurrentLocation(); + if (p.parseString(&text) || p.parseGreater()) + return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter " + "'layout' as `cutegen::layout`"); + return {}; + } + return getChecked([&] { return p.emitError(loc); }, p.getContext(), desc, + cute::LayoutAttr::get(p.getContext(), *layout)); +} +void SmemDescViewType::print(AsmPrinter &p) const { + p << '<' << getDesc() << ", "; + p.printString(cutegen::to_string(getLayout().getRef())); + p << '>'; +} +LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, + cute::LayoutAttr) { + if (!llvm::isa(desc)) + return emitError() << "expected SmemDescType, SmemDescCircularSM103Type, " + "SmemDescCircularSM107Type or SmemDescSM107Type"; + return success(); +} + +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +// ``; the compiler prints +// `` for copy_bits 0, which is accepted too. +Type CopyAtomSIMTSyncCopyType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto src = cute::AddressSpace::generic, dst = src; + if (p.parseLess() || p.parseType(t)) + return {}; + // `src_space` / `dst_space` groups; one alone is what the compiler prints + // when the other is generic (or copy_bits is 0), so it is read too. + auto spaces = [&]() -> ParseResult { + if (succeeded(p.parseOptionalKeyword("dst_space"))) + return failure(p.parseEqual() || field(p, dst)); + if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) + return failure(); + if (succeeded(p.parseOptionalComma())) + return keyField(p, "dst_space", dst); + return success(); + }; + if (succeeded(p.parseOptionalComma())) { + OptionalParseResult n = p.parseOptionalInteger(bits); + if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) + return {}; + if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) + return {}; + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, + src, dst); +} +void CopyAtomSIMTSyncCopyType::print(AsmPrinter &p) const { + auto generic = cute::AddressSpace::generic; + bool spaces = getSrcSpace() != generic || getDstSpace() != generic; + p << '<' << getValType(); + if (getCopyBits() || spaces) + p << ", " << getCopyBits() << " b"; + if (spaces) + p << ", src_space = " << stringifyEnum(getSrcSpace()); + if (getDstSpace() != generic) + p << ", dst_space = " << stringifyEnum(getDstSpace()); + p << '>'; +} +LogicalResult CopyAtomSIMTSyncCopyType::verify(EmitFn emitError, Type, int bits, + cute::AddressSpace, + cute::AddressSpace) { + if (bits != 0 && !isPow2AtLeast8(bits)) + return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 " + "(>= 8), but got " + << bits; + return success(); +} + +// ``. +Type CopyAtomSIMTAsyncCopyType::parse(AsmParser &p) { + Type t; + int bits = 0; + LoadCacheMode cache; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; + return get(p.getContext(), t, cache, orElementBits(bits, t)); +} +void CopyAtomSIMTAsyncCopyType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", cache = " << stringifyEnum(getCache()) << ", " + << getCopyBits() << " b>"; +} + +// ``. +Type CopyAtomDsmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t)); +} +void CopyAtomDsmemStoreType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getCopyBits() << " b>"; +} +LogicalResult CopyAtomDsmemStoreType::verify(EmitFn emitError, Type, int bits) { + if (bits != 0 && bits != 32 && bits != 64 && bits != 128) + return emitError() + << "expects copy_bits to be one of (0, 32, 64, 128), but got " + << bits; + return success(); +} + +// Bulk copies: ``. +Type CopyAtomBulkCopyG2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mcast = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mcast", mcast) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mcast); +} +void CopyAtomBulkCopyG2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMcast()) + p << " mcast = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mask = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mask", mask) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mask); +} +void CopyAtomBulkCopyS2GType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMask()) + p << " mask = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + p.parseGreater()) + return {}; + return get(p.getContext(), t, bits); +} +void CopyAtomBulkCopyS2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + p << '>'; +} + +// g2r, r2g, r2s, s2r: ``. +static LogicalResult verifyCopyBits(EmitFn emitError, int bits) { + if (!isPow2AtLeast8(bits)) + return emitError() + << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; + return success(); +} + +Type CopyAtomG2RType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto prefetch = L2PrefetchSize::NONE; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = LoadCacheMode::always; + auto shared = SharedSpace::CTA; + bool invariant = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l2_prefetch_size", prefetch) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "load_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || + optKeyField(p, "invariant", invariant) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, prefetch, evict, + cache, shared, invariant); +} +void CopyAtomG2RType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l2_prefetch_size", getL2PrefetchSize(), L2PrefetchSize::NONE); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "load_cache_mode", getLoadCacheMode(), LoadCacheMode::always); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + if (getInvariant()) + p << " invariant = true"; + p << '>'; +} +LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind scope, + L2PrefetchSize prefetch, + CacheEvictionPriority evict, + LoadCacheMode cache, SharedSpace, + bool invariant) { + if (failed(verifyCopyBits(emitError, bits))) + return failure(); + if (invariant && (order != MemOrderKind::WEAK || scope != MemScopeKind::CTA || + prefetch != L2PrefetchSize::NONE || + evict != CacheEvictionPriority::EVICT_NORMAL || + cache != LoadCacheMode::always)) + return emitError() + << "invariant_load is not allowed when the other non-default memory " + "features (e.g., mem_order) are set to non-default values"; + if (order != MemOrderKind::WEAK && cache != LoadCacheMode::always) + return emitError() + << "expects load_cache_mode to be 'always' when mem_order is not " + "'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +Type CopyAtomR2GType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = StoreCacheMode::write_back; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "store_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, evict, cache, shared); +} +void CopyAtomR2GType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "store_cache_mode", getStoreCacheMode(), + StoreCacheMode::write_back); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind, + CacheEvictionPriority, + StoreCacheMode cache, SharedSpace) { + if (failed(verifyCopyBits(emitError, bits))) + return failure(); + if (order != MemOrderKind::WEAK && cache != StoreCacheMode::write_back) + return emitError() + << "expects store_cache_mode to be 'write_back' when mem_order is " + "not 'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +template static Type parseSharedCopy(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) + return {}; + return T::getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, shared); +} +template static void printSharedCopy(AsmPrinter &p, T type) { + p << '<' << type.getValType() << " copy_bits = " << type.getCopyBits(); + printIfNot(p, "mem_order", type.getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", type.getMemScope(), MemScopeKind::CTA); + printIfNot(p, "shared_space", type.getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +Type CopyAtomR2SType::parse(AsmParser &p) { + return parseSharedCopy(p); +} +void CopyAtomR2SType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} +Type CopyAtomS2RType::parse(AsmParser &p) { + return parseSharedCopy(p); +} +void CopyAtomS2RType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} + +//===----------------------------------------------------------------------===// +// Multimem atoms: `, ORDER, SCOPE>`, positional. Parsing +// takes each field by its kind, which also reads what the compiler prints +// when it drops a default field before a set one. +//===----------------------------------------------------------------------===// + +namespace { +struct Multimem { + int bits = 0; + ReductionKind red = ReductionKind::ADD; + LdReduceAccPrecisionKindAttr acc; + MemOrderKind order = MemOrderKind::RELAXED; + MemScopeKind scope = MemScopeKind::SYS; +}; + +ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, + bool hasAcc) { + if (p.parseLess() || p.parseType(t)) + return failure(); + while (succeeded(p.parseOptionalComma())) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + OptionalParseResult bits = p.parseOptionalInteger(m.bits); + if (bits.has_value()) { + if (failed(*bits) || p.parseKeyword("b")) + return failure(); + continue; + } + if (hasAcc && succeeded(p.parseOptionalLess())) { + LdReduceAccPrecisionKind kind; + if (field(p, kind) || p.parseGreater()) + return failure(); + m.acc = LdReduceAccPrecisionKindAttr::get(p.getContext(), kind); + continue; + } + if (p.parseKeyword(&kw)) + return failure(); + if (auto r = symbolizeReductionKind(kw); r && hasRed) + m.red = *r; + else if (auto o = symbolizeMemOrderKind(kw)) + m.order = *o; + else if (auto s = symbolizeMemScopeKind(kw)) + m.scope = *s; + else + return p.emitError(loc, "unexpected `") << kw << "`"; + } + return p.parseGreater(); +} + +// Prints the fields up to the last one that is not default. +void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, + bool hasRed, bool hasAcc) { + int last = m.scope != MemScopeKind::SYS ? 4 + : m.order != MemOrderKind::RELAXED ? 3 + : m.acc ? 2 + : hasRed && m.red != ReductionKind::ADD ? 1 + : bitsSet ? 0 + : -1; + p << '<' << t; + if (last >= 0) + p << ", " << m.bits << " b"; + if (hasRed && last >= 1) + p << ", " << stringifyEnum(m.red); + if (hasAcc && last >= 2 && m.acc) { + p << ", "; + p.printStrippedAttrOrType(m.acc); + } + if (last >= 3) + p << ", " << stringifyEnum(m.order); + if (last >= 4) + p << ", " << stringifyEnum(m.scope); + p << '>'; +} +} // namespace + +Type CopyAtomSIMTMultimemStType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) + return {}; + return get(p.getContext(), t, m.bits, m.order, m.scope); +} +void CopyAtomSIMTMultimemStType::print(AsmPrinter &p) const { + Multimem m{ + getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, false, false); +} + +Type CopyAtomSIMTMultimemRedType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) + return {}; + // An unset width is the element's, at least 32 bits (the f16x2 form). + int bits = m.bits ? m.bits : std::max(elementBits(t), 32); + return get(p.getContext(), t, bits, m.red, m.order, m.scope); +} +void CopyAtomSIMTMultimemRedType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, /*bitsSet=*/true, true, false); +} + +Type CopyAtomSIMTMultimemLdReduceType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) + return {}; + return get(p.getContext(), t, m.bits, m.red, m.acc, m.order, m.scope); +} +void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), + getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, true, true); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory atoms: `` +//===----------------------------------------------------------------------===// + +static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, + unsigned &rep) { + return failure(p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || + countOf(p, "bit", bit) || p.parseComma() || repeat(p, rep)); +} +static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, + unsigned rep) { + p << '<' << t << ", " << dp << " DP, " << bit << " bit, x" << rep; +} +// An optional `, word`. +static ParseResult parseFlag(AsmParser &p, StringRef word, UnitAttr &out) { + if (failed(p.parseOptionalComma())) + return success(); + if (p.parseKeyword(word)) + return failure(); + out = UnitAttr::get(p.getContext()); + return success(); +} + +static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, + unsigned rep) { + if (dp != 16 && dp != 32) + return emitError() + << "expects the number of data paths to be one of [16,32], but got " + << dp; + if (bit != 32 && bit != 64 && bit != 128 && bit != 256) + return emitError() << "expects the number of bits to be one of " + "[32,64,128,256], but got " + << bit; + if (rep < 1 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[1,2,4,8,16,32,64,128], but got " + << rep; + if (dp == 32 && bit != 32) + return emitError() + << "only 32b patterns are supported when the number of data paths " + "is 32, but got " + << bit; + if (dp == 16 && bit == 256 && rep >= 64) + return emitError() + << "a repetition of 64 or 128 is not supported when the number of " + "data paths is 16 and the number of bits is 256"; + if (dp == 16 && bit == 128 && rep == 128) + return emitError() + << "a repetition of 128 is not supported when the number of data " + "paths is 16 and the number of bits is 128"; + return success(); +} + +Type CopyAtomSM100TmemLoadType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr pack; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, pack); +} +void CopyAtomSM100TmemLoadType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getPack16b() ? ", pack16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +Type CopyAtomSM100TmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr expand; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "expand16b", expand) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, expand); +} +void CopyAtomSM100TmemStoreType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getExpand16b() ? ", expand16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +// `<..., OP[, nan][, half_split_off = N[ : T]]>`. +Type CopyAtomSM10xTmemLoadRedType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr nan; + IntegerAttr split; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) + return {}; + while (succeeded(p.parseOptionalComma())) { + if (!nan && !split && succeeded(p.parseOptionalKeyword("nan"))) { + nan = UnitAttr::get(p.getContext()); + continue; + } + if (p.parseKeyword("half_split_off") || p.parseEqual() || + p.parseAttribute(split)) + return {}; + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, nan, split); +} +void CopyAtomSM10xTmemLoadRedType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getNan()) + p << ", nan"; + if (getHalfSplitOff()) + p << ", half_split_off=" << getHalfSplitOff(); + p << '>'; +} +LogicalResult +CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, + unsigned rep, TmemLoadRedOp op, + UnitAttr nan, IntegerAttr split) { + if (!(t.isF32() || t.isInteger(32))) + return emitError() << "expect val_type to be u32, s32, or f32, but got '" + << t << "'"; + if (nan && !t.isF32()) + return emitError() << "expect val_type to be f32 when nan is specified"; + if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && + !t.isF32()) + return emitError() + << "expect val_type to be f32 when red_op is minabs or maxabs"; + if (bit != 32) + return emitError() << "only 32b patterns are supported, but got " << bit; + if (rep < 2 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[2,4,8,16,32,64,128], but got " + << rep; + if (split && dp != 16) + return emitError() + << "expect half_split_off to be set only when mode is 16dp 32bit"; + return success(); +} + +// `<..., OP[, red[, nan]]>`. +Type CopyAtomSM107TmemLoadSPCompressType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr red, nan; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op) || + parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, red, nan); +} +void CopyAtomSM107TmemLoadSPCompressType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getRed()) + p << ", red"; + if (getNan()) + p << ", nan"; + p << '>'; +} +LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify( + EmitFn emitError, Type t, int dp, int bit, unsigned rep, TmemLoadRedOp, + UnitAttr red, UnitAttr nan) { + if (dp != 32 || bit != 32) + return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " + << dp << " and " << bit; + if (!t.isF32()) + return emitError() << "expect val_type to be f32, but got '" << t << "'"; + if (rep < 4 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expect the number of repetitions to be one of " + "[4,8,16,32,64,128], but got " + << rep; + if (nan && !red) + return emitError() << "expect `red` to be true when `nan` is true"; + return success(); +} + +// The UTCCP shapes of tcgen05.cp. +static bool isUtccp(int dp, int bit, CopyS2TBroadcast b) { + using B = CopyS2TBroadcast; + return (b == B::none && (dp == 4 || dp == 128) && + (bit == 128 || bit == 256)) || + (b == B::x4 && dp == 32 && bit == 128) || + ((b == B::lw_0213 || b == B::lw_0123) && dp == 64 && bit == 128); +} + +// ``. +Type CopyAtomSM100CopyS2TType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || + p.parseComma() || countOf(p, "cta", cta)) + return {}; + if (succeeded(p.parseOptionalComma()) && field(p, bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, bcast); +} +void CopyAtomSM100CopyS2TType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() + << " bit, " << getNumCta() << " cta"; + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, + int bit, int cta, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) + return emitError() << "Expect cta is 1/2 but got: " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "The given UTCCP params: dp " << dp << ", bits " + << bit << ", broadcast " << stringifyEnum(b) + << " is unsupported or illegal."; + return success(); +} + +// ``. +Type CopyAtomSM100S2TCopyV2Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + cute::MajorMode major; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "num_dp", dp) || + nextKeyField(p, "num_bit", bit) || nextKeyField(p, "num_cta", cta) || + nextKeyField(p, "smem_major", major)) + return {}; + if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, major, bcast); +} +void CopyAtomSM100S2TCopyV2Type::print(AsmPrinter &p) const { + p << '<' << getValType() << ", num_dp = " << getNumDp() + << ", num_bit = " << getNumBit() << ", num_cta = " << getNumCta() + << ", smem_major = " << stringifyEnum(getSmemMajor()); + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", broadcast = " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, + int bit, int cta, + cute::MajorMode, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) + return emitError() << "expects cta to be 1 or 2, but got " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "expects valid UTCCP params, but got dp " << dp + << ", bits " << bit << ", broadcast " + << stringifyEnum(b); + return success(); +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +Type CopyAtomLdsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + LdsmSzPattern sz; + int n; + UnitAttr trans; + if (p.parseLess() || keyField(p, "val_type", t) || + nextKeyField(p, "mode", mode) || nextKeyField(p, "sz_pattern", sz) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, sz, n, trans); +} +void CopyAtomLdsmType::print(AsmPrinter &p) const { + p << "" : ", n>"); +} + +Type CopyAtomStsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + int n; + UnitAttr trans; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, n, trans); +} +void CopyAtomStsmType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", mode = "; + p.printStrippedAttrOrType(getMode()); + p << ", num_matrices = " << getNumMatrices() + << (getTranspose() ? ", t>" : ", n>"); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +static LogicalResult verifySparsity(EmitFn emitError, int sparsity) { + if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && + sparsity != 16) + return emitError() + << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " + << sparsity; + return success(); +} +// `'; +} +LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, + int, TmaLoadMode, int, + cute::StrideType, bool, + cute::LayoutType, bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int sparsity = 1, bits; + TmaStoreMode mode; + cute::StrideType stride; + cute::LayoutType gbasis; + bool override = false, noOob = false; + if (parseTmaHead(p, t, &sparsity, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || + optKeyField(p, "override", override) || + optKeyField(p, "no_fully_oob_tile", noOob) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + sparsity, bits, mode, stride, gbasis, override, noOob); +} +void CopyAtomTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), getSparsity(), getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()); + printTmaTail(p, getGStride(), getTmaGbasis()); + printTrue(p, "override", getOverride()); + printTrue(p, "no_fully_oob_tile", getNoFullyOobTile()); + p << '>'; +} +LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, + int, TmaStoreMode, cute::StrideType, + cute::LayoutType, bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaReduceType::parse(AsmParser &p) { + Type t; + int bits; + TmaStoreMode mode; + ReductionKind kind; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "kind", kind) || nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mode, kind, stride, gbasis); +} +void CopyAtomTmaReduceType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()) + << ", kind = " << stringifyEnum(getKind()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +Type CopyAtomIm2ColTmaLoadType::parse(AsmParser &p) { + Type t; + int bits, ctas; + cute::StrideType stride; + cute::LayoutType gbasis; + bool mcast = false; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "num_cta", ctas) || + nextKeyField(p, "g_stride", stride) || optKeyField(p, "mcast", mcast) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, ctas, stride, mcast, gbasis); +} +void CopyAtomIm2ColTmaLoadType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", num_cta = " << getNumCta() << ", g_stride = "; + p.printStrippedAttrOrType(getGStride()); + printTrue(p, "mcast", getMcast()); + if (getTmaGbasis()) { + p << " tma_gbasis = "; + p.printStrippedAttrOrType(getTmaGbasis()); + } + p << '>'; +} + +Type CopyAtomIm2ColTmaStoreType::parse(AsmParser &p) { + Type t; + int bits; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || + nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, stride, gbasis); +} +void CopyAtomIm2ColTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +// The TMA data format of an element type when none is given. +std::optional +mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { + if (auto s = llvm::dyn_cast(t)) + t = s.getPhysicalType(); + if (t.isF16()) + return TmaDataFormat::F16_RN; + if (t.isBF16()) + return TmaDataFormat::BF16_RN; + if (t.isF32()) + return TmaDataFormat::F32_RN; + if (t.isTF32()) + return TmaDataFormat::TF32_RN; + if (t.isF64()) + return TmaDataFormat::F64_RN; + if (!t.isIntOrFloat()) + return std::nullopt; + unsigned bits = t.getIntOrFloatBitWidth(); + bool isSigned = t.isSignedInteger(); + if (bits == 4) + return TmaDataFormat::U4; + if (bits <= 8) + return TmaDataFormat::U8; + if (bits == 16) + return TmaDataFormat::U16; + if (bits == 32) + return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; + if (bits == 64) + return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; + return std::nullopt; +} + +namespace { +// `<[kind, ]T, copy_bits = N, tma_gbasis = L[, tma_format = F]>`. +template +ParseResult parseNonExecTma(AsmParser &p, Kind *kind, Type &t, int &bits, + cute::LayoutType &gbasis, + std::optional &format) { + if (p.parseLess()) + return failure(); + if (kind && (field(p, *kind) || p.parseComma())) + return failure(); + if (p.parseType(t) || nextKeyField(p, "copy_bits", bits) || + nextKeyField(p, "tma_gbasis", gbasis)) + return failure(); + format = defaultTmaDataFormat(t); + if (succeeded(p.parseOptionalComma())) { + // Empty for an element type without a format, as the compiler prints it. + if (p.parseKeyword("tma_format") || p.parseEqual()) + return failure(); + if (succeeded(p.parseOptionalGreater())) { + format = std::nullopt; + return success(); + } + TmaDataFormat f; + if (field(p, f)) + return failure(); + format = f; + } + return p.parseGreater(); +} +template void printNonExecTma(AsmPrinter &p, T type) { + p << type.getValType() << ", copy_bits = " << type.getCopyBits() + << ", tma_gbasis = "; + p.printStrippedAttrOrType(type.getTmaGbasis()); + p << ", tma_format = "; + if (type.getTmaFormat()) + p << stringifyEnum(*type.getTmaFormat()); + p << '>'; +} +LogicalResult verifyNonExecTma(EmitFn emitError, int bits) { + if (bits == 0) + return emitError() << "invalid copy_bits, got 0"; + return success(); +} +} // namespace + +#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + KIND kind; \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, \ + t, bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<' << stringifyEnum(getKind()) << ", "; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +#define NON_EXEC_TMA(TYPE) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, \ + bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<'; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaLoadType, TiledTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaReduceType, ReductionKind) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecIm2ColTmaLoadType, Im2ColTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, + GatherScatterTmaLoad) +NON_EXEC_TMA(CopyAtomNonExecTiledTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExecIm2ColTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExec2DScatter4TmaStoreType) +#undef NON_EXEC_TMA_WITH_KIND +#undef NON_EXEC_TMA + +//===----------------------------------------------------------------------===// +// MMA atoms +//===----------------------------------------------------------------------===// + +Type UniversalFmaAtomType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseLParen() || p.parseType(a) || p.parseComma() || p.parseType(b) || + p.parseRParen() || p.parseArrow() || p.parseType(c) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); +} +void UniversalFmaAtomType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() + << " >"; +} +LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, + Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + MMAIntOverflowAttr overflow; + BinaryOpAttr op; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c)) + return {}; + if (succeeded(p.parseOptionalComma()) && + (keyField(p, "int_overflow", overflow) || + (succeeded(p.parseOptionalComma()) && keyField(p, "binary_op", op)))) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, overflow, op); +} +void MmaAtomSM80Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + printAttrField(p, "int_overflow", getIntOverflow()); + printAttrField(p, "binary_op", getBinaryOp()); + p << " >"; +} +LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type, MMAIntOverflowAttr, + BinaryOpAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80SparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + SparseMetadataFormat format; + MMAIntOverflowAttr overflow; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "md_format", format)) + return {}; + if (succeeded(p.parseOptionalComma()) && + keyField(p, "int_overflow", overflow)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, format, overflow); +} +void MmaAtomSM80SparseType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", md_format = " << stringifyEnum(getSparseMetadataFormat()); + printAttrField(p, "int_overflow", getIntOverflow()); + p << " >"; +} +LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, + Type, SparseMetadataFormat, + MMAIntOverflowAttr) { + return verifyShapeMNK(emitError, shape); +} + +Type MmaAtomSM89Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); +} +void MmaAtomSM89Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << " >"; +} +LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM90Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + cute::MajorMode aMajor, bMajor; + Type a, b, c; + MmaFragKind frag; + MMAIntOverflowAttr overflow; + UnitAttr aNeg, bNeg; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, aMajor, bMajor) || p.parseComma() || elemTypes(p, a, b, c) || + p.parseComma() || parseFragKind(p, MmaFragKind::rmem, frag)) + return {}; + while (succeeded(p.parseOptionalComma())) { + SMLoc at = p.getCurrentLocation(); + if (succeeded(p.parseOptionalKeyword("a_neg"))) + aNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("b_neg"))) + bNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("int_overflow"))) { + if (p.parseEqual() || field(p, overflow)) + return {}; + } else { + p.emitError(at, "expected `int_overflow`, `a_neg`, or `b_neg`"); + return {}; + } + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + aMajor, bMajor, a, b, c, frag, overflow, aNeg, bNeg); +} +void MmaAtomSM90Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " + << stringifyEnum(getBMajor()) << "), "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", frag_kind = " << fragKindText(getAFragKind()); + printAttrField(p, "int_overflow", getIntOverflow()); + if (getANeg()) + p << ", a_neg"; + if (getBNeg()) + p << ", b_neg"; + p << '>'; +} +LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, + cute::MajorMode, cute::MajorMode, Type, + Type, Type, MmaFragKind, + MMAIntOverflowAttr, UnitAttr, UnitAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM120BlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + int vec; + Type a, b, c, sf; + bool useTV; + if (p.parseLess() || parseShapeMNK(p, shape) || + nextKeyField(p, "vec_size", vec) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || + nextKeyField(p, "use_sf_layout_TV", useTV) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + vec, a, b, c, sf, useTV); +} +void MmaAtomSM120BlockScaledType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", vec_size = " << getVecSize() << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", sf_type = " << getSfType() + << ", use_sf_layout_TV = " << (getUseSfLayout_TV() ? "true" : "false") + << '>'; +} +LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, + cute::ShapeAttr shape, int, + Type, Type, Type, Type, + bool) { + return verifyShapeMNK(emitError, shape); +} + +// UMMA atoms: one grammar, fields present by kind (see CuteNVGPUTypes.td). +namespace { +struct Umma { + bool sf, sp, bs, sm107; + cute::ShapeAttr shape; + int numCta = 1; + cute::MajorMode aMajor, bMajor; + Type a, b, c, sfType, eType; + SparseMetadataFormat metadata = SparseMetadataFormat::tid; + MmaFragKind frag = MmaFragKind::smem_desc; + int scaleOrVec = 0; + Arch arch = Arch::sm_100; + MmaCollectorOp aCollector = MmaCollectorOp::discard, + bCollector = MmaCollectorOp::discard; + MMAIntOverflowAttr overflow; +}; + +ParseResult parseUmma(AsmParser &p, Umma &u) { + if (p.parseLess() || parseShapeMNK(p, u.shape) || + nextKeyField(p, "num_cta", u.numCta) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || + elemTypes(p, u.a, u.b, u.c)) + return failure(); + if (u.sf && nextKeyField(p, "sf_type", u.sfType)) + return failure(); + if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) + return failure(); + if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) + return failure(); + if (p.parseComma() || parseFragKind(p, MmaFragKind::tmem, u.frag) || + nextKeyField(p, u.bs ? "vec_size" : "c_scale_exp", u.scaleOrVec)) + return failure(); + bool comma = succeeded(p.parseOptionalComma()); + if (comma && u.bs && !u.sm107 && + succeeded(p.parseOptionalKeyword("arch_promote"))) { + if (p.parseEqual() || field(p, u.arch)) + return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (u.sm107) { + if (!comma) + return p.parseComma(); + if (p.parseKeyword("ab_collector_op") || p.parseEqual() || + pairOf(p, u.aCollector, u.bCollector)) + return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (comma && keyField(p, "int_overflow", u.overflow)) + return failure(); + return p.parseGreater(); +} + +void printUmma(AsmPrinter &p, const Umma &u) { + p << '<'; + printShapeMNK(p, u.shape); + p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) + << ", " << stringifyEnum(u.bMajor) << "), "; + printElemTypes(p, u.a, u.b, u.c); + if (u.sf) + p << ", sf_type = " << u.sfType; + if (u.sp && !u.sf) + p << ", e_type = " << u.eType; + if (u.sp) + p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); + p << ", frag_kind = " << fragKindText(u.frag) + << (u.bs ? ", vec_size = " : ", c_scale_exp = ") << u.scaleOrVec; + if (u.bs && !u.sm107 && (u.arch != Arch::sm_100 || u.overflow)) + p << ", arch_promote = " << stringifyEnum(u.arch); + if (u.sm107) + p << ", ab_collector_op = (" << stringifyEnum(u.aCollector) << ", " + << stringifyEnum(u.bCollector) << ")"; + printAttrField(p, "int_overflow", u.overflow); + p << '>'; +} + +// `sourced` picks the atom's wording of the tensor-memory A rule. +LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, + MmaFragKind frag, cute::MajorMode aMajor, + int cScaleExp = 0, bool sourced = false) { + if (failed(verifyShapeMNK(emitError, shape))) + return failure(); + if (numCta != 1 && numCta != 2) + return emitError() << "expects `num_cta` equals to 1 or 2, but got " + << numCta; + if (cScaleExp > 15) + return emitError() << "expects `c_scale_exp` in range [0, 15], but got " + << cScaleExp; + if (frag == MmaFragKind::tmem && aMajor != cute::MajorMode::k) + return emitError() + << (sourced + ? "a operand must be k-major when sourced from tmem" + : "a operand must be of major-mode k when coming from tmem"); + return success(); +} +} // namespace + +Type MmaAtomSM100UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, + cute::ShapeAttr shape, int numCta, + cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, + MmaFragKind frag, int cScaleExp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.arch, u.overflow); +} +void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, Arch, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +Type MmaAtomSM100UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.arch, u.overflow); +} +void MmaAtomSM100UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, Arch, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor); +} + +Type MmaAtomSM107UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); +} +void MmaAtomSM107UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMAType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, MmaFragKind frag, int cScaleExp, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); +} +void MmaAtomSM107UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MmaCollectorOp, MmaCollectorOp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.aCollector, u.bCollector, + u.overflow); +} +void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +Type MmaAtomSM107UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); +} +void MmaAtomSM107UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +//===----------------------------------------------------------------------===// +// Dialect +//===----------------------------------------------------------------------===// + +// `!cute_nvgpu.iN[]` is the compiler's alias of `!cute.iN[]`. +Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { + SMLoc loc = parser.getCurrentLocation(); + StringRef mnemonic; + Type type; + OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); + if (result.has_value()) + return succeeded(*result) ? type : Type(); + StringRef width = mnemonic; + if (width.consume_front("i") && !width.empty() && + llvm::all_of(width, llvm::isDigit)) { + std::string spec = ("!cute." + mnemonic).str(); + int64_t divisor; + if (succeeded(parser.parseOptionalLess())) { + if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || + parser.parseGreater()) + return {}; + spec += ""; + } + return mlir::parseType(spec, getContext()); + } + parser.emitError(loc) << "unknown type `" << mnemonic << "` in dialect `" + << getNamespace() << "`"; + return {}; +} + +void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { + if (succeeded(generatedTypePrinter(type, printer))) + return; + llvm_unreachable("unexpected non-CuteNVGPU type"); +} + +void CuteNVGPUDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" + >(); +} diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h new file mode 100644 index 00000000..19d8cb14 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h @@ -0,0 +1,38 @@ +//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect -------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h.inc" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.h.inc" + +namespace mlir::cutlass_compiler::cute_nvgpu { +/// Parses `true` or `false`. +FailureOr parseBool(AsmParser &parser); +/// The TMA data format a non-executable TMA atom takes for an element type +/// when none is given, if there is one. +std::optional defaultTmaDataFormat(Type elementType); +} // namespace mlir::cutlass_compiler::cute_nvgpu + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.h.inc" + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td new file mode 100644 index 00000000..5f6c6680 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td @@ -0,0 +1,36 @@ +//===- CuteNVGPUDialect.td - CuTe NVIDIA GPU dialect -------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def CuteNVGPU_Dialect : Dialect { + let name = "cute_nvgpu"; + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let summary = "NVIDIA GPU operations and atom types used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation, type and attribute registry of + nvidia-cutlass-dsl 4.8.0, embedded in the proprietary CuTe-DSL compiler. + The type parser is hand-written only to accept `!cute_nvgpu.iN`, the + compiler's alias of `!cute.iN`. + }]; + let extraClassDeclaration = [{ + ::mlir::Type parseType(::mlir::DialectAsmParser &parser) const override; + void printType(::mlir::Type type, + ::mlir::DialectAsmPrinter &printer) const override; + }]; + let useDefaultAttributePrinterParser = 1; +} + +class CuteNVGPU_Op traits = []> + : Op; + +class CuteNVGPU_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td new file mode 100644 index 00000000..a583a57e --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td @@ -0,0 +1,683 @@ +// The cute_nvgpu dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_nvgpu_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. +// The cute enums the nvgpu bindings repeat are the cute dialect's; their +// nvgpu attributes are in CuteNVGPUAttrs.td, so that -gen-enum-decls +// over this file emits the nvgpu enums only. + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" +include "mlir/IR/EnumAttr.td" + +def CuteNVGPU_BinaryOpEnum : I32EnumAttr<"BinaryOp", "Binary operation for single-bit MMA operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyG2S", "Fields stored in Bulk Load Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2G", "Fields stored in Bulk Store Copy Atom type", [ + I32EnumAttrCase<"byte_mask", 4, "byte_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2S", "Fields stored in Bulk CTA to Cluster Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"cta_rank", 5, "cta_rank"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreEnum : I32EnumAttr<"AtomCopyFieldDsmemStore", "Fields stored in CopyAtomDsmemStoreType", [ + I32EnumAttrCase<"mbar_ptr", 8, "mbar_ptr"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalEnum : I32EnumAttr<"AtomCopyFieldLoadGlobal", "Fields stored in CopyAtomG2RType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExec2DGather4TmaLoad", "Fields stored in CopyAtomNonExec2DGather4TmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExec2DScatter4TmaStore", "Fields stored in CopyAtomNonExec2DScatter4TmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaLoad", "Fields stored in CopyAtomNonExecIm2ColTmaLoadType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaStore", "Fields stored in CopyAtomNonExecIm2ColTmaStoreType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaLoad", "Fields stored in CopyAtomNonExecTiledTmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaReduce", "Fields stored in CopyAtomNonExecTiledTmaReduceType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaStore", "Fields stored in CopyAtomNonExecTiledTmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalEnum : I32EnumAttr<"AtomCopyFieldStoreGlobal", "Fields stored in CopyAtomR2GType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadEnum : I32EnumAttr<"AtomCopyFieldTmaLoad", "Fields stored in Tma Load Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceEnum : I32EnumAttr<"AtomCopyFieldTmaReduce", "Fields stored in Tma Reduce Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreEnum : I32EnumAttr<"AtomCopyFieldTmaStore", "Fields stored in Tma Store Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseEnum : I32EnumAttr<"AtomMmaFieldSM80Sparse", "Fields stored in MMA Atom types for SM80", [ + I32EnumAttrCase<"e", 0, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM90Enum : I32EnumAttr<"AtomMmaFieldSM90", "Fields stored in MMA Atom types for SM90", [ + I32EnumAttrCase<"accum_c", 0, "accum_c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaled", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaledSparse", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100Enum : I32EnumAttr<"AtomMmaFieldSM100", "Fields stored in MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"disable_output_lane", 6, "disable_output_lane"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseEnum : I32EnumAttr<"AtomMmaFieldSM100Sparse", "Fields stored in sparse MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM120BlockScaled", "Fields stored in MMA Atom types for SM120 (block scaled)", [ + I32EnumAttrCase<"sf_a", 0, "sf_a">, + I32EnumAttrCase<"sf_b", 1, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "Cute Cache Eviction Priority kind", [ + I32EnumAttrCase<"EVICT_NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"EVICT_FIRST", 1, "evict_first">, + I32EnumAttrCase<"EVICT_LAST", 2, "evict_last">, + I32EnumAttrCase<"EVICT_UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NO_ALLOCATE", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastEnum : I32EnumAttr<"CopyS2TBroadcast", "Broadcast modes for the different utccp instructions", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"lw_0213", 1, "lw_0213">, + I32EnumAttrCase<"lw_0123", 2, "lw_0123">, + I32EnumAttrCase<"x4", 3, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_GatherScatterTmaLoadEnum : I32EnumAttr<"GatherScatterTmaLoad", "The various kinds of TMA loads in gather/scatter mode", [ + I32EnumAttrCase<"sm_100", 0, "sm_100">, + I32EnumAttrCase<"sm_100_multicast", 1, "sm_100_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ArchEnum : I32EnumAttr<"Arch", "NVGpu arch number like sm_103", [ + I32EnumAttrCase<"sm_52", 520, "sm_52">, + I32EnumAttrCase<"sm_70", 700, "sm_70">, + I32EnumAttrCase<"sm_75", 750, "sm_75">, + I32EnumAttrCase<"sm_80", 800, "sm_80">, + I32EnumAttrCase<"sm_86", 860, "sm_86">, + I32EnumAttrCase<"sm_90", 900, "sm_90">, + I32EnumAttrCase<"sm_100", 1000, "sm_100">, + I32EnumAttrCase<"sm_103", 1030, "sm_103">, + I32EnumAttrCase<"sm_107", 1070, "sm_107"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_Im2ColTmaLoadEnum : I32EnumAttr<"Im2ColTmaLoad", "The various kinds of TMA loads in im2col mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "CuTe L2 prefetch size options", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdReduceAccPrecisionKindEnum : I32EnumAttr<"LdReduceAccPrecisionKind", "multimem ld_reduce accumulation precision kind", [ + I32EnumAttrCase<"F16", 0, "F16">, + I32EnumAttrCase<"F32", 1, "F32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdsmSzPatternEnum : I32EnumAttr<"LdsmSzPattern", "LDSM's sz pattern, describing the bit size", [ + I32EnumAttrCase<"u16", 0, "u16">, + I32EnumAttrCase<"u4to8", 1, "u4to8">, + I32EnumAttrCase<"s4to8", 2, "s4to8">, + I32EnumAttrCase<"u2to4", 3, "u2to4">, + I32EnumAttrCase<"s2to4", 4, "s2to4">, + I32EnumAttrCase<"u4x16p64to8", 5, "u4x16p64to8">, + I32EnumAttrCase<"u6x16p32to8", 6, "u6x16p32to8">, + I32EnumAttrCase<"u8", 7, "u8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LoadCacheModeEnum : I32EnumAttr<"LoadCacheMode", "Cache modes for the load instructions", [ + I32EnumAttrCase<"always", 0, "always">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"last_use", 3, "last_use">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"CONSTANT", 7, "constant">, + I32EnumAttrCase<"VOLATILE", 8, "volatile"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "Cute Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaCollectorOpEnum : I32EnumAttr<"MmaCollectorOp", "Enums for the mma collector op", [ + I32EnumAttrCase<"discard", 0, "discard">, + I32EnumAttrCase<"lastuse", 1, "lastuse">, + I32EnumAttrCase<"fill", 2, "fill">, + I32EnumAttrCase<"use", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaFragKindEnum : I32EnumAttr<"MmaFragKind", "Enums for the mma frag type", [ + I32EnumAttrCase<"smem_desc", 0, "smem_desc">, + I32EnumAttrCase<"sparse_smem_desc", 1, "sparse_smem_desc">, + I32EnumAttrCase<"tmem", 2, "tmem">, + I32EnumAttrCase<"tmem_ws", 3, "tmem_ws">, + I32EnumAttrCase<"tmem_e", 4, "tmem_e">, + I32EnumAttrCase<"tmem_sf", 5, "tmem_sf">, + I32EnumAttrCase<"rmem", 6, "rmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ReductionKindEnum : I32EnumAttr<"ReductionKind", "Op for the TMASTORE instruction", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MIN", 1, "MIN">, + I32EnumAttrCase<"MAX", 2, "MAX">, + I32EnumAttrCase<"INC", 3, "INC">, + I32EnumAttrCase<"DEC", 4, "DEC">, + I32EnumAttrCase<"AND", 5, "AND">, + I32EnumAttrCase<"OR", 6, "OR">, + I32EnumAttrCase<"XOR", 7, "XOR"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "CuTe Shared Space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SparseMetadataFormatEnum : I32EnumAttr<"SparseMetadataFormat", "Metadata format used in sparse MMA", [ + I32EnumAttrCase<"tid", 0, "tid"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_StoreCacheModeEnum : I32EnumAttr<"StoreCacheMode", "Cache modes for the store instructions", [ + I32EnumAttrCase<"write_back", 0, "write_back">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"write_through", 3, "write_through">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TiledTmaLoadEnum : I32EnumAttr<"TiledTmaLoad", "The various kinds of TMA loads in tiled mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaDataFormatEnum : I32EnumAttr<"TmaDataFormat", "Bits 74-71 of the TMA descriptor", [ + I32EnumAttrCase<"U8", 0, "U8">, + I32EnumAttrCase<"U16", 1, "U16">, + I32EnumAttrCase<"U32", 2, "U32">, + I32EnumAttrCase<"S32", 3, "S32">, + I32EnumAttrCase<"U64", 4, "U64">, + I32EnumAttrCase<"S64", 5, "S64">, + I32EnumAttrCase<"F16_RN", 6, "F16_RN">, + I32EnumAttrCase<"F32_RN", 7, "F32_RN">, + I32EnumAttrCase<"F32_FTZ_RN", 8, "F32_FTZ_RN">, + I32EnumAttrCase<"F64_RN", 9, "F64_RN">, + I32EnumAttrCase<"BF16_RN", 10, "BF16_RN">, + I32EnumAttrCase<"U4", 11, "U4">, + I32EnumAttrCase<"U4_UNPACK_U8", 12, "U4_UNPACK_U8">, + I32EnumAttrCase<"U6_UNPACK_U8", 13, "U6_UNPACK_U8">, + I32EnumAttrCase<"TF32_RN", 14, "TF32_RN">, + I32EnumAttrCase<"TF32_FTZ_RN", 15, "TF32_FTZ_RN"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaLoadModeEnum : I32EnumAttr<"TmaLoadMode", "Modes for the TMA load instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"w128", 2, "w128">, + I32EnumAttrCase<"w", 3, "w">, + I32EnumAttrCase<"gather4", 4, "gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaStoreModeEnum : I32EnumAttr<"TmaStoreMode", "Modes for the TMA store instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"scatter4", 2, "scatter4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemAllocModeEnum : I32EnumAttr<"TmemAllocMode", "Modes for the Tmem allocation", [ + I32EnumAttrCase<"Interleaved", 0, "Interleaved">, + I32EnumAttrCase<"NonInterleaved", 1, "NonInterleaved">, + I32EnumAttrCase<"Duplicated", 2, "Duplicated">, + I32EnumAttrCase<"SfDuplicated_4x1", 3, "SfDuplicated_4x1">, + I32EnumAttrCase<"SfDuplicated_2x2", 4, "SfDuplicated_2x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemLoadRedOpEnum : I32EnumAttr<"TmemLoadRedOp", "Reduce operation for the Tmem load instruction", [ + I32EnumAttrCase<"max", 0, "max">, + I32EnumAttrCase<"maxabs", 1, "maxabs">, + I32EnumAttrCase<"min", 2, "min">, + I32EnumAttrCase<"minabs", 3, "minabs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM90Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_BinaryOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_GatherScatterTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_Im2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdReduceAccPrecisionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdsmSzPatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LoadCacheModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaCollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaFragKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaDataFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaLoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemAllocModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemLoadRedOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td b/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td new file mode 100644 index 00000000..f4c09d12 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td @@ -0,0 +1,2012 @@ +// The cute_nvgpu dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_nvgpu_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute_nvgpu.arch.mma.SM107.umma_block_scaled operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_sparse operand #3: no probe type accepted; any type allowed + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, "allowable descriptor types for S2T src operand">; +def CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA : Type($_self))">, "allowable descriptor types for SM107 Blockscaled UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand : Type($_self))">, "allowable descriptor types for SM107 UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA2 : Type($_self))">, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3 : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def CuteNVGPU_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def CuteNVGPU_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def CuteNVGPU_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def CuteNVGPU_CuTeMemrefToMemorySpacesSmem : Type($_self))">, "CuTe memref to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA : Type($_self))">, "CuTe pointer to allowable types for SM100 UMMA C/D operands in spaces (tmem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2 : Type, "CuTe pointer to allowable types for SM100 UMMA E operand in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA : Type, "CuTe pointer to allowable types for SM107 UMMA SF A/B operands in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA : Type($_self))">, "CuTe pointer to allowable types for SM90 MMA A/B operands in spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte : Type($_self))">, "CuTe pointer to global memory with 64-byte alignment">; +def CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment : Type($_self))">, "CuTe pointer to global memory with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesRmem : Type($_self))">, "CuTe pointer to memory spaces (rmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmem : Type($_self))">, "CuTe pointer to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe : Type($_self))">, "CuTe pointer to memory spaces (smem) or CuTe pointer to memory spaces (gmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte2 : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 128">; +def CuteNVGPU_CuTePointerToMemorySpacesTmem : Type($_self))">, "CuTe pointer to memory spaces (tmem)">; +def CuteNVGPU_CuTePointerToSharedMemoryWith8Byte : Type($_self))">, "CuTe pointer to shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment with byte alignment 1">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2 : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment">; +def CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory : Type($_self))">, "CuTe pointer to shared or distributed shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def CuteNVGPU_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def CuteNVGPU_CuteCoordType : Type($_self))">, "Cute coord type">; +def CuteNVGPU_CuteShapeType : Type($_self))">, "Cute shape type">; +def CuteNVGPU_CuteSm107SmemDescType : Type($_self))">, "Cute sm107.smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescType : Type($_self))">, "Cute smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescType::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescViewType : Type($_self))">, "Cute smem_desc_view type">; +def CuteNVGPU_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def CuteNVGPU_Integer : Type($_self))">, "integer">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom : Type($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp.">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp. or Tiled copy atom type.">; +def CuteNVGPU_SM90PointerToTMATensorDescriptor : Type($_self))">, "SM90+ pointer to TMA tensor descriptor">; +def CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self))">, "SM90+ TMA (Tensor Memory Accelerator) load atom type via cp.async.bulk. or SM90+ TMA im2col mode load atom type via cp.async.bulk.tensor. or SM90+ TMA (Tensor Memory Accelerator) store atom type via cp.async.bulk. or SM90+ TMA im2col mode store atom type via cp.async.bulk.tensor. or SM90+ TMA store with atomic reduction atom type via cp.reduce.async.bulk. or Type interface for non-executable TMA atom types">; +def CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::IndexType>($_self))">, "signless integer or index or CuTe Layout Type">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA : Type($_self))">, "The non-executable copy atom type for TMA loads in 2D gather4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2 : Type($_self))">, "The non-executable copy atom type for TMA stores in 2D scatter4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3 : Type($_self))">, "The non-executable copy atom type for TMA loads in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4 : Type($_self))">, "The non-executable copy atom type for TMA stores in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5 : Type($_self))">, "The non-executable copy atom type for TMA loads in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6 : Type($_self))">, "The non-executable copy atom type for TMA reduce in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7 : Type($_self))">, "The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "The non-executable copy atom type for TMA loads in tiled mode or The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "Type interface for non-executable TMA atom types">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf : Type($_self))">, "vector of 32-bit signless integer values of length 1/2/4/8/16/32/64/128">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2 : Type($_self))">, "vector of 32-bit signless integer values of length 4/8">; +def CuteNVGPU_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; + +// Attribute constraints the bindings name but do not describe. +def CuteNVGPU_ArchCopySM100CopyS2TBitAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TCtaAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{4, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumBAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{32, 64, 128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{16, 32}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumRepAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2, 4, 8, 16, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledABTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B operands"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledSFTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B scale factors"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_TmaLoadOrStoreModeAttr : Attr($_self)">, "TMA load or store mode attribute"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class CuteNVGPU_InferredOp traits = []> + : CuteNVGPU_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_Op<"arch.alloc_rmem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate a per-thread private buffer on the stack, with its size + determined by either the provided element count or the static layout. + + Examples: + + ```mlir + %c32_i32 = arith.constant 32 : i32 + %0 = cute_nvgpu.arch.alloc_rmem(%c32_i32) : i32 -> !cute.ptr + + // %static_layout : !cute.layout<"(8,4):(1,8)"> + %1 = cute_nvgpu.arch.alloc_rmem(%static_layout) : !cute.layout<"(8,4):(1,8)"> -> !cute.ptr + + // %dynamic_layout : !cute.layout<"(?,4):(1,?)"> + %2 = cute_nvgpu.arch.alloc_rmem(%dynamic_layout) : !cute.layout<"(?,4):(1,?)"> -> !cute.ptr + ``` + }]; + let arguments = (ins + CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesRmem:$ptr + ); +} +def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_Op<"arch.alloc_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate static shared memory based on either a specified + element count or a static layout. + + Examples: + + ```mlir + // size provided by element count + %0 = cute_nvgpu.arch.alloc_smem(1024) : !cute.ptr> + + // size provided by static layout + %1 = cute_nvgpu.arch.alloc_smem("(8,4):(1,8)") : !cute.ptr> + ``` + }]; + let arguments = (ins + I32Attr:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyG2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_g2s", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_g2s` operation performs a bulk load from global + to shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the source data + in global memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `multicast_mask` operand is used when multicasting. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %gmem = ... : !cute.ptr> + %smem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + + // Without multicast mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 + + // With multicast mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + Optional:$multicast_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2GOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2g", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2g` operation performs a bulk load from shared + to global memory. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the sourced data + in shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the estination data + in global memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `byte_mask` operand is specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %gmem = ... : !cute.ptr> + + // Without byte mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 + + // With byte mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 byte_mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + Optional:$byte_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2s"> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2s` operation performs a bulk load from shared + cta to shared cluster. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the source data + in shared memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `cta_rank` operand specifies the dst cta rank in cluster for BLKCP. + + The `size` attr specifies the bytes copied by BLKCP, aligned to 16 bytes. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %dsmem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + %cta = ... : i32 + + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2s( + %smem : !cute.ptr>, + %dsmem : !cute.ptr>, + %bar : !cute.ptr>, + %cta : i32 + ) size = 128 + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + I32:$cta_rank, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100CopyS2TOp : CuteNVGPU_Op<"arch.copy.SM100.copy_s2t"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand:$src, + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_ArchCopySM100CopyS2TDpAttr:$dp, + CuteNVGPU_ArchCopySM100CopyS2TBitAttr:$bits, + CuteNVGPU_ArchCopySM100CopyS2TCtaAttr:$cta, + CuteNVGPU_CopyS2TBroadcastAttr:$broadcast + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tma_load", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_load` operation performs a tensor load from global + to shared memory. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - w128 for activation-stationary convolution MMA mode (requires multicast) + - w for weight-stationary convolution MMA mode (requires multicast) + - gather4 for gather mode + + The attribute `num_cta` specifies the scope of the instruction. + + The `src_desc` operand is a 64-byte aligned pointer to the source tensor in + global memory. + + The `dsmem_data_addr` is a 128-byte aligned pointer to the destination data + address in shared memory. + + The `dsmem_bar_addr` is an 8-byte aligned pointer to the barrier in shared + or distributed shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `multicast_mask` operand is used when multicasting. + + The `offsets` operand holds offsets for im2col, w128, or w modes. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + Variadic:$coord, + Optional:$multicast_mask, + Variadic:$offsets, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaReduceOp : CuteNVGPU_Op<"arch.copy.SM100.tma_reduce", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_reduce` operation performs a tensor store from shared memory + to global with reduction. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + + The attribute `Kind` indicates which kind of reduce op is selected. It can be: + + - ADD + - MIN + - MAX + - INC + - DEC + - AND + - OR + - XOR + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode, + CuteNVGPU_ReductionKindAttr:$kind + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tma_store", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_store` operation performs a tensor store from shared memory + to global. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - scatter4 for scatter mode + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmemLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_load", [MemoryEffects<[MemRead]>]> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$pack_16 + ); + let results = (outs + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$dst + ); +} +def CuteNVGPU_ArchCopySM100TmemStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_store"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$expand_16 + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaLoadOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_load_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_load_override` operation performs a TMA tensor load + from global to shared memory with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA load operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Coordinates are passed as zeros (override form ignores them). + + Example: + + Suppose we want to load a 2D tile from a tensor with logical sizes + `{S0, S1}` and per-mode element strides `{1, ES1}` (mode 0 is contiguous, + so its stride is not passed). Let `EBITS` be the value type's bit-width + and `elemsPer16B = 128 / EBITS`. Mode 0's size is in `tensor_size[0]`, + mode 1's size is in `tensor_size[1]`, and mode 1's stride is split between + `lower_stride[0]` and the nibble at bits `[0:4]` of `upper_stride`. + + The encoding (matching `CopyAtomTmaLoadType::lowerOverrideToArch`) is: + + ```mlir + // ---- tensor_size: trunc(S_i - 1), one entry per mode ---- + %s0_m1 = arith.subi %S0_i32, %c1_i32 : i32 // S0 - 1 + %s1_m1 = arith.subi %S1_i32, %c1_i32 : i32 // S1 - 1 + %ts0 = arith.trunci %s0_m1 : i32 to i16 + %ts1 = arith.trunci %s1_m1 : i32 to i16 + + // ---- stride in 16B units: ES_i (element stride, i64) >> log2(elemsPer16B) + %sh = arith.constant log2(elemsPer16B) : i64 + %s1_16b = arith.shrui %ES1_i64, %sh : i64 // mode 1, stride in 16B + + // ---- lower_stride[idx]: low 32 bits of stride_in_16B (idx = mode - 1) ---- + %ls1 = arith.trunci %s1_16b : i64 to i32 // idx = 0 (mode 1) + + // ---- upper_stride: packed nibbles, OR-accumulated, truncated to i16 ---- + // nibble_i = (stride_in_16B[mode i] >> 32) & 0xF + // upper |= nibble_i << (4 * (i - 1)) + %c32 = arith.constant 32 : i64 + %c0xF = arith.constant 15 : i64 + %hi_1 = arith.shrui %s1_16b, %c32 : i64 // upper bits of mode 1 + %nib_1 = arith.andi %hi_1, %c0xF : i64 + %sh_1 = arith.constant 0 : i64 // 4 * (1 - 1) = 0 + %shf_1 = arith.shli %nib_1, %sh_1 : i64 + %acc = arith.ori %c0_i64, %shf_1 : i64 // accumulate over modes + %us = arith.trunci %acc : i64 to i16 + + // ---- the override TMA load itself ---- + cute_nvgpu.arch.copy.SM107.tma_load_override( + %tma_desc : !cute.ptr>, // src TMA descriptor + %dst_smem : !cute.ptr>, // dst data buffer + %dst_bar : !cute.ptr>, // mbarrier + %override_addr : !cute.ptr>, // runtime base addr override + [%c0_i32, %c0_i32] : i32, i32) // coords (ignored: zeros) + tensor_size = %ts0, %ts1 // (size - 1) per mode, i16 + lower_stride = %ls1 // strides for modes 1..N-1, in 16B units, low 32b (i32) + upper_stride = %us // packed upper 4 bits of up to 4 strides (i16) + cache_policy = %policy // L2 eviction policy (i64) + mode = num_cta = 1 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$multicast_mask, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaStoreOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_store_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_store_override` operation performs a TMA tensor store + from shared memory to global with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA store operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Example + The example of tma_load_override applies here except that tma_store_override + doesn't have the mbar argument. + + Coordinates are passed as zeros (override form ignores them). + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchGetDynamicSmemOp : CuteNVGPU_Op<"arch.get_dyn_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation acquires the pointer to the dynamic shared memory, whose size + is determined at the kernel launch time. + }]; + let arguments = (ins + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchGetDynamicSmemSizeOp : CuteNVGPU_Op<"arch.get_dyn_smem_size", [Pure]> { + let description = [{ + This operation gets the size in bytes of the dynamic shared memory that was + specified at kernel launch time. This can be used for bounds checking during + shared memory allocation. + }]; + let arguments = (ins + ); + let results = (outs + I32:$size + ); +} +def CuteNVGPU_ArchMakeWarpUniformOp : CuteNVGPU_Op<"arch.make_warp_uniform", [Pure]> { + let description = [{ + This operation is to reach a similar goal as canonical_warp_idx_sync() in cutlass C++, which is + typically used to hint to ptxas to use uniform register to store current value so as to avoid + redundant R2URs in mainloop. It's actually a shortcut usage of `nvvm.shfl.sync idx 31, val, 0, -1`. + + Example: + + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_id_uniform = cute_nvgpu.arch.make_warp_uniform %warp_id : i32 + ``` + And this IR will be lowered to: + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_full_mask = arith.constant -1 : i32 + %c0_i32 = arith.constant 0 : i32 + %warp_id_uniform = nvvm.shfl.sync idx %c0_i32, %warp_id, %c0_i32, %warp_full_mask : i32 -> i32 + ``` + }]; + let arguments = (ins + I32:$val + ); + let results = (outs + I32:$res + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled(%A, %B, %C, %sf_a, %sf_b, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledSparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B and supports + structured sparsity. Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `e_op` operand + provides the sparse metadata that describes the sparsity positional information of the matrix, + enabling efficient computation with structured sparse matrices. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse(%A, %B, %C, %sf_a, %sf_b, %E, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, !cute.ptr>, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$e_op, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMAOp : CuteNVGPU_Op<"arch.mma.SM107.umma"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma` operation performs a matrix multiply- + accumulate operation. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma(%A, %B, %C, %idesc, %accum) + { num_cta = 1, ab_type = (f8E5M2, f8E5M2), shape_k = 64, ab_collector_op = (discard, discard) } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + I32:$idesc, + I1:$accum, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2:$disable_output_lane, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMASparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_sparse` operation performs a matrix + multiply-accumulate operation where matrix A is structured sparse. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. Operand E holds sparse + metadata that reside in tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_sparse(%A, %B, %C, %E, %idesc, %accum) + {num_cta = 1 : i32, scale = 0 : i32, ab_type = (f16, f16), shape_k = 32 : i32, ab_collector_op = (discard, discard)} + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, + !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2:$e, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM120BlockScaledOp : CuteNVGPU_Op<"arch.mma.SM120.block_scaled", [AttrSizedOperandSegments]> { + let description = [{ + The `cute_nvgpu.arch.mma.SM120.block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for A/B/C/D/SFA/SFB and accumulator are always in registers. + + C is always F32 under current SM120 arch, so there is no `c_type` here. + + See the file cutlass/include/cute/arch/mma_sm120.hpp in the CUTLASS repository + for more information. + + Examples: + ```mlir + cute_nvgpu.arch.mma.SM120.block_scaled + A[%a0, %a1, %a2, %a3] B[%b0, %b1] C[%c0, %c1, %c2, %c3] + sfA(%scaleAData) sfB(%scaleBData) + {shape_MNK=<"(16,8,32)">, vec_size = 32, a_type = f4E2M1FN, b_type = f4E2M1FN, sf_type = f8E8M0FNU} + : A[i32, i32, i32, i32] B[i32, i32] C[f32, f32, f32, f32] + sfA(i8) sfB(i8) -> (f32, f32, f32, f32) + ``` + }]; + let arguments = (ins + Variadic:$opA, + Variadic:$opB, + Variadic:$opC, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfA, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfB, + Optional:$byte_id_a, + Optional:$byte_id_b, + Cute_ShapeAttr:$shape_MNK, + I32Attr:$vec_size, + DefaultValuedAttr:$thread_id_a, + DefaultValuedAttr:$thread_id_b, + CuteNVGPU_SM120BlockScaledABTypeAttr:$a_type, + CuteNVGPU_SM120BlockScaledABTypeAttr:$b_type, + CuteNVGPU_SM120BlockScaledSFTypeAttr:$sf_type + ); + let results = (outs + Variadic:$res + ); +} +def CuteNVGPU_ArchPrefetchTmaDescOp : CuteNVGPU_Op<"arch.prefetch_tma_desc"> { + let description = [{ + The `arch.prefetch_tma_desc` operation does prefetch TMA descriptor using generic addressing + (i.e. no specific state space: const or param). + + Example: + + ```mlir + cute_nvgpu.arch.prefetch_tma_desc(%tma_a) : (!cute.ptr>) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$tma_descriptor_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100AllocTmemOp : CuteNVGPU_Op<"arch.sm100.alloc_tmem"> { + let summary = "The `arch.sm100.alloc_tmem` operation performs dynamic allocation of TMEM."; + let description = [{ + It dynamically allocates storage on TMEM with requested `n_cols` and stores + the allocated TMEM pointer into the shared memory buffer at the shared memory + location specified by `dst_ptr`. + + This operation is blocking, meaning it will wait until enough memory is available. + The action writting pointer value into shared memory is not synchronized. It's + user's responsibility to synchronize to make sure it's visible to all threads in the CTA. + + The allocation may fail if one of following conditions is true: + 1. The requested allocation size is not a multiple of 32 columns or greater + than 512 columns or not a power of 2, or + 2. If an attempt is made to allocate after executing the + `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit` operation. + + This operation must be issued by a single fully active warp of the CTA and + must not be issued by more than one warp simultaneously. For repeated + allocations, the same warp must be used to issue all allocations. All + `alloc_tmem` operations must be issued by the same warp. + + For the allocation of 2 CTAs, one warp from each of the peer CTAs must + collectively perform the allocation, and the issuing warp must make sure that + peer CTA is launched and is still active. The participating warps from both + CTAs should have the same logical warp ID. There is no explicit need for + synchronization between these 2 CTAs before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `n_cols`: The number of columns to be allocated. The unit of allocation is + 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. The number of columns allocated should not increase + between any two allocations in the execution order within the CTA. Failing to + meet these conditions may lead to undefined behavior. + - `dst_ptr`: Pointer to shared memory where the result TMEM pointer will be + written. It should have a natural alignment of 32-bits. + - `is_two_cta`: Specifies the number of CTAs involved in the allocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_1] + : i32, !cute.ptr> + + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_2] + : i32, !cute.ptr> + ``` + }]; + let arguments = (ins + I32:$n_cols, + CuteNVGPU_CuTePointerToMemorySpacesSmem:$dst_ptr, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100DeallocTmemOp : CuteNVGPU_Op<"arch.sm100.dealloc_tmem"> { + let description = [{ + The `arch.sm100.dealloc_tmem` operation deallocates the Tensor Memory + specified by the TMEM address `tmem_ptr` and slice count. The address + `tmem_ptr` must point to a previously allocated TMEM. If the + deallocation fails, a runtime error occurs. + + This operation should only be performed by a single fully active warp within + the CTA. It should not be carried out by multiple warps simultaneously. The + warp ID that does alloc and dealloc can be different, but all `dealloc_tmem` + operations must be issued by the same warp. + + For the 2 CTAs deallocation, one warp from each of the peer CTAs must + collectively perform the de-allocation, and the issuing warp must make sure + that peer CTA is launched and is still active. The warps from both CTAs should + have the same logical warp ID. Prior explicit synchronization between the two + CTAs is not required before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `tmem_ptr`: The base address of the TMEM buffer being deallocated. + - `n_cols`: The number of columns to be deallocated. The unit of deallocation + is 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. Failing to meet these conditions may lead to undefined + behavior. + - `is_two_cta`: Specifies the number of CTAs involved in the deallocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_1] + : !cute.ptr>, i32 + + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_2] + : !cute.ptr>, i32 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr, + I32:$n_cols, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp : CuteNVGPU_Op<"arch.sm100.relinquish_tmem_alloc_permit"> { + let description = [{ + This `arch.sm100.relinquish_tmem_alloc_permit` specifies that the CTA of the + executing thread is relinquishing the right to allocate Tensor Memory so that + other CTAs can allocate. + + Once this operation is executed, any subsequent attempts by the same CTA to + allocate TMEM are considered illegal. + + Example: + + ```mlir + cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit [cta_1] + ``` + }]; + let arguments = (ins + UnitAttr:$is_two_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RetrieveTmemPtrOp : CuteNVGPU_Op<"arch.sm100.retrieve_tmem_ptr", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `arch.sm100.retrieve_tmem_ptr` operation is a convenient shorthand for + loading the allocated TMEM pointer from shared memory and casting it into a + TMEM pointer. It includes a correctness check for the TMEM pointer and + returns a null pointer if the TMEM pointer is invalid. + + Example: + + ```mlir + %tmem_ptr = cute_nvgpu.arch.sm100.retrieve_tmem_ptr(%res_shm_ptr) + : !cute.ptr> -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$smem_ptr + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr + ); +} +def CuteNVGPU_AtomCopyMakeExecTmaOp : CuteNVGPU_InferredOp<"atom.make_exec_tma", [Pure]> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$input, + UnitAttr:$override, + UnitAttr:$no_fully_oob_tile + ); + let results = (outs + CuteNVGPU_CopyAtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DGather4TmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_gather4_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled gather4 mode (.tile::gather4) + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_coord_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_GatherScatterTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DScatter4TmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_scatter4_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in tiled scatter4 mode (.tile::scatter4) + 2. the TMA tensor mapping the logical coordinates of the scatter index layout to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_index_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_load. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_Im2ColTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_store. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_TiledTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaReduceOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_reduce", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA reduce in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_ReductionKindAttr:$kind, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA store in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeS2TCopyOp : CuteNVGPU_InferredOp<"atom.make_s2t_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling an utccp copy atom + instance with a tmem memref. + + Example: + ```mlir + %copy_s2t_atom = cute_nvgpu.atom.make_s2t_copy() : !cute_nvgpu.atom.s2t_copy + %tmem_memref = cute.memref.alloca() : !cute.memref, "(128,32):(65536,1)"> + %0 = cute_nvgpu.atom.make_s2t_copy(%copy_s2t_atom, %tmem_memref) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"(1,(128,8)):(0,(1,128))">, + tiler_mn = <"[128:1;8:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom:$copy_s2t_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomCopyMakeTmemCopyOp : CuteNVGPU_InferredOp<"atom.make_tmem_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a load tmem copy atom + instance with a tmem memref. + + Example: + ```mlir + // %tmem_memref : !cute.memref, "(128,32):(65536,1)"> + %load_atom = cute.make_atom() : !cute_nvgpu.atom.tmem_load + %0 = cute_nvgpu.atom.make_tmem_copy(%load_atom, %tmem_memref) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"((32,4),(32,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);32:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomGetCoordTensorOp : CuteNVGPU_InferredOp<"atom.get_coord_tensor", [SameVariadicOperandSize, Pure]> { + let description = [{ + This operation creates a get tma coord atom instance from copy atom and shapes. + The atom will be called before algo copy(tma copy) to provide tma coord for algo copy. + In Cute C++, copy_atom cache stride info during make_tiled_copy and the stride + info is associated with shapes. shapes in argument will make a congruent comparison + with cache first and then construct tma coord with stride. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_coord_tensor(%4, %5) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<128, 64>) + -> !cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)"> + ``` + + ```mlir + %6 = cute_nvgpu.atom.get_coord_tensor(%4, %5, %2, %3) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<"(500, 128)">, + !cute.int_tuple<"(0, 0)">, + !cute.layout<"(128, 256): (1@0, 1@1) + -> !cute.coord_tensor<"(0,0)", "(500,128):(1@1,1@0) o (0,0) o (128,256):(1@0,1@1)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + CuteNVGPU_CuteShapeType:$shape, + Optional:$offset_b, + Optional:$layout_b + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp : CuteNVGPU_InferredOp<"atom.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewSM107Op : CuteNVGPU_InferredOp<"atom.sm107.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetValueOp : CuteNVGPU_Op<"atom.get_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_value(%4) : (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>) -> i1 + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyAttr:$field + ); + let results = (outs + AnyType:$result + ); +} +def CuteNVGPU_AtomMakeTmaResidueTensorOp : CuteNVGPU_InferredOp<"atom.make_tma_residue_tensor", [Pure]> { + let description = [{ + Builds the per-TMA-mode "remaining elements" coord tensor that + `cute.copy` consumes on the override path. The op takes a non-executable + TMA atom (whose type carries `tma_gbasis`) and the corresponding gmem + tensor, and produces a coord tensor whose + + - iter is gmem tensor's shape combined and ordered according to tma gbasis. + - layout has the same shape as the tma tensor while the stride is negated. + + The residue tensor tracks the remaining shape to copy during TMA override, + e.g, slicing the residue tensor with a cta coordinate gets the remaining + shape for it given the negated stride. + + Example: + ```mlir + %residue = cute_nvgpu.atom.make_tma_residue_tensor(%non_exec_atom, %gmem) + : (!cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?):(?,1)">) + -> !cute.coord_tensor<"(?,?)", "(?,?):(-1@1,-1@0)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$residue + ); +} +def CuteNVGPU_AtomSetValueOp : CuteNVGPU_InferredOp<"atom.set_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %2 = cute_nvgpu.atom.set_value(%0, %no_accum) : + (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyType:$value, + AnyAttr:$field + ); + let results = (outs + CuteNVGPU_AtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomTmaPartitionOp : CuteNVGPU_InferredOp<"atom.tma_partition", [Pure]> { + let description = [{ + tma_partition is the new TMA routine for the dynamic and preferred cluster feature. + tma_partition op requires below arguments: + 1. tma_atom; + 2. cta_coord in the multicast dimension; + 3. cta_layout in the multicast dimension; + 4. smem_memref; + 5. variadic CuteViewTypeInterface tensors (one or more coord_tensor / memref), + covering the following cases: + 5.a: for regular TMA load/store, a single TMA tensor as input + 5.b: for tile::gather4 mode, a 2D TMA coord tensor plus the corresponding + gmem index tensor + and it produces: + 1. partitioned_smem_memref; + 2. one partitioned coord_tensor / memref per input, with the following cases: + 2.a: for regular TMA load/store, one partitioned TMA tensor + 2.b: for tile::gather4 mode, the partitioned TMA coord tensor plus the + partitioned gmem index tensor + + The return values are the src and dst arguments of algo.copy(atom, src, dst). + The second and the third input arguments can be runtime values, so the multicast + can be runtime determined. + + This operation is also compatible for the old static cluster TMA routines. + You only need to feed the cta_coord and cta_layout with static values, + then you can get compile-time multicast computation. + + The target_tensors operand is variadic and accepts any CuteViewTypeInterface + type (coord_tensor or memref). When a single tensor is provided, brackets + are omitted for backward compatibility. When multiple tensors are provided, + they are grouped in brackets. + + Example (single tensor): + ```mlir + %res_smem, %res_gmem = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, %gtensor + ) : (!cute_nvgpu.atom.tma_load, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, S<3,4,3>, "(((128,16),1,4),8):(((64,1),0,16),8192)">, + !cute.coord_tensor<"(0,?,0)", "(((128,16),1,4),?,?,?):(((1@1,1@0),0,16@0),256@1,64@0,1@2)">) + ``` + + Example (multiple tensors): + ```mlir + %res_smem, %res_gmem0, %res_gmem1 = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, [%gtensor0, %gtensor1] + ) : (!cute_nvgpu.atom.tma_load<...>, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + -> (!cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType:$tma_atom, + CuteNVGPU_CuteCoordType:$cta_coord, + CuteNVGPU_CuTeLayoutType:$cta_layout, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$smem_tensor, + Variadic:$target_tensors + ); + let results = (outs + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$res_smem_tensor, + Variadic:$res_target_tensors + ); +} +def CuteNVGPU_CastTMADescToIntegerOp : CuteNVGPU_Op<"cast_tma_desc_to_integer", [Pure]> { + let summary = "cast_tma_desc_to_integer cast a tma_desc into an integer."; + let description = [{ + Example: + ```mlir + %23 = cute_nvgpu.cast_tma_desc_to_integer(%11 : !cute.ptr>) -> i64 + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$src + ); + let results = (outs + CuteNVGPU_Integer:$dst + ); +} +def CuteNVGPU_CopyTMADescriptorOp : CuteNVGPU_Op<"copy_tma_desc"> { + let description = [{ + This operation copies the TMA descriptor from a non-executable TMA atom to the pointer. + The pointer could be a smem or gmem pointer and should be aligned to at least the size of TMA descriptor. + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_GetGridConstantPointerOp : CuteNVGPU_InferredOp<"get_grid_constant_pointer", [Pure]> { + let description = [{ + 'cute_nvgpu.get_grid_constant_pointer' takes a grid constant value argument + from a kernel and returns a `cute.ptr` whose pointee type matches the + input. + + Before device ABI conversion, the input may be an intermediate result that + function finalization will promote to a grid-constant kernel argument. + Direct kernel arguments are verified immediately; intermediate provenance + is validated when the physical kernel signature is finalized. The result + type and alignment are fully inferred from the input type by + `InferTypeOpInterface`. + Example (TMA descriptor): + func.func @kernel( + %tma_desc_a : !cute_nvgpu.tma_descriptor_tiled {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %tma_desc_a_ptr = cute_nvgpu.get_grid_constant_pointer (%tma_desc_a) : + !cute.ptr>; + ... + } + + Example (arbitrary value type): + func.func @kernel( + %params : !llvm.struct<(i64, i64)> {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %params_ptr = cute_nvgpu.get_grid_constant_pointer (%params) : + !cute.ptr, generic>; + ... + } + }]; + let arguments = (ins + AnyType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerType:$result + ); +} +def CuteNVGPU_GetTmaDescAddrOp : CuteNVGPU_Op<"get_tma_desc_addr", [Pure]> { + let description = [{ + The `cute_nvgpu.get_tma_desc_addr` operation accepts an input copy atom, + extracts the address within copy atom, then returns it as the address + of current copy atom's TMA desc in a single SSA value. The address within + input copy atom should have global/generic address space, and should be aligned to 64B. + Example: + + ```mlir + %1 = cute_nvgpu.get_tma_desc_addr(%cpy_atom : !cute_nvgpu.atom.tma_load) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$src + ); + let results = (outs + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$ptr + ); +} +def CuteNVGPU_MakeGMMASmemDescOp : CuteNVGPU_Op<"make_gmma_smem_desc", [Pure]> { + let description = [{ + The `cute_nvgpu.make_gmma_smem_desc` operation accepts an input cute.ptr alongside a cute.layout and a major attr, + then constructs the shared memory descriptor and returns it. The layout must be describing + the buffer pointed to by the input pointer and must carry valid swizzle information. + For GMMA, there are 4 supported swizzle variants: + + ``` + swizzle | cooresponding enum in gmma smem desc + S<0, 4, 3> | SWIZZLE_NONE + S<1, 4, 3> | SWIZZLE_32B + S<2, 4, 3> | SWIZZLE_64B + S<3, 4, 3> | SWIZZLE_128B + ``` + The cute.ptr must carry shared address space and must be aligned to 16B. + + Example: + + ```mlir + %1 = cute_nvgpu.make_gmma_smem_desc(%arg1 : !cute.ptr, S<3,4,3>>) + layout = <"((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA:$src, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_MakeTmaDescIm2ColAtOp : CuteNVGPU_Op<"make_tma_desc_im2col_at"> { + let description = [{ + In-place variant of Cute_MakeTmaDescIm2ColOp. The descriptor is stored + into the address passed as the last operand. + + Operands (identical semantics to the im2col op) + - All operands of MakeTmaDescIm2ColOp (see above). + - Value tma_desc_addr – destination address for the descriptor. + + Optional attributes (identical semantics to the im2col op) + - num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_im2col_at( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt, + %tma_desc_addr) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.ptr>) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescIm2ColOp : CuteNVGPU_Op<"make_tma_desc_im2col", [Pure]> { + let summary = "Create a TMA descriptor in im2col mode, suitable for convolutional forward passes."; + let description = [{ + Operands + - Value gmem_view, smem_layout, cta_value_tile – as in the tiled op. + - Value lower_corner_whd / upper_corner_whd – start & end (W,H,D) + positions of the + receptive-field window. + - Value lower_padding_whd / upper_padding_whd – padding applied + before/after each dim. + - Value stride_whd – traversal stride in WHD. + - Value lower_srt / stride_srt – split-K traversal + parameters (SRT). + + Optional attributes + - num_multicast : i32 + - tma_format : CuteTmaDataFormatAttr + + Result + - Cute pointer to the generated descriptor. + + Example: + ```mlir + %desc = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeTmaDescTiledAtOp : CuteNVGPU_Op<"make_tma_desc_tiled_at"> { + let summary = "In-place version of Cute_MakeTmaDescTiledOp."; + let description = [{ + Operands (identical semantics to the Tiled op) + - Value gmem_view – Global-memory tensor view. + - Value smem_layout – SMEM box layout. + - Value cta_value_tile – CTA-value-tile mapping. + - Value tma_desc_addr – Cute pointer that names the memory location + where the TMA descriptor should be written. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes (identical semantics to the Tiled op) + - mode, num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_tiled_at(%a_tensor, %a_smem_clayout, %a_cta_v_tile, %tma_desc_addr) + <{mode = }> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64,4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled_at(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile, %existing_tma_desc) + <{mode = num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %tma_desc_addr %traversal_stride) + <{mode = }> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.ptr>, + !cute.int_tuple<"(1,?,?,?)">) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescTiledOp : CuteNVGPU_Op<"make_tma_desc_tiled", [Pure]> { + let summary = "Create a TMA descriptor in tiled load mode."; + let description = [{ + Operands + - Value gmem_view – Cute view that describes the tensor in global memory. + - Value smem_layout – Composed/regular Cute layout that defines the + rectangular box in shared memory (SMEM tile). + - Value cta_value_tile – Cute layout that maps CTA threads to elements + within the SMEM tile. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes + - mode : CuteTmaLoadModeAttr or CuteTmaStoreModeAttr (defaults to #cute_nvgpu.tma_load_mode) + - num_multicast : i32 (defaults to 1; 1 ⇒ no multicast) + - tma_format : CuteTmaDataFormatAttr (element format written to + bits 74–71 of the HW TMA + descriptor; if omitted an + appropriate default is deduced) + + Result + - Cute pointer to the freshly created TMA-descriptor object. + + Example: + ```mlir + %tma_desc_a = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %a_smem_clayout, %a_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode}> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64),4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">) -> !cute.ptr> + + %tma_desc_sfa = cute_nvgpu.make_tma_desc_tiled(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1)):((((16,4),512),(0,1)),0,(1,512))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">) -> !cute.ptr> + + %tma_desc_strided_c = cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %traversal_stride) + <{mode = #cute_nvgpu.tma_load_mode}> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.int_tuple<"(1,?,?,?)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadOrStoreModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeUMMASM107SmemDescOp : CuteNVGPU_Op<"sm107.make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm107.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.sm107.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSm107SmemDescType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM103Op : CuteNVGPU_InferredOp<"sm103.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM107Op : CuteNVGPU_InferredOp<"sm107.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescOp : CuteNVGPU_Op<"make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_PrefetchTmaDescriptorOp : CuteNVGPU_Op<"prefetch_tma_desc"> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom + ); + let results = (outs + ); +} +def CuteNVGPU_UpdateTMADescOp : CuteNVGPU_Op<"update_tma_desc"> { + let description = [{ + This operation updates the TMA descriptor in smem or gmem with the information from given gmem tensor. + Specifically, the following fields of the TMA descriptor will be updated: + 1. the global tensor base address will be updated with the base address of the global memory tensor. + 2. the tensor shapes will be updated with the shape of the global memory tensor. + 3. the tensor strides will be updated with the stride of the global memory tensor. + Other fields of the TMA descriptor will be left unchanged. + The TMA descriptor pointer specifies the address of the tma descriptor to be updated. + It could be a smem or gmem pointer. The pointer should be aligned to at least 64B. + Example: + ```mlir + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %smem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (1,?,?)">, + !cute.ptr>) + -> () + + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %gmem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.ptr>) + -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td new file mode 100644 index 00000000..817af564 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td @@ -0,0 +1,750 @@ +//===- CuteNVGPUTypes.td - CuTe NVIDIA GPU types -----------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and verifier strings, its typed Python +// type classes, and probing its type parser and printer); do not edit by hand. +// +// Every cute_nvgpu type of the compiler is here, with its parameters in the +// compiler's order and C++ types. Parameters the compiler parses with custom +// directives (the `MxNxK` shape, `frag_kind`, `D DP`, `xR`, `t|n`, the sm90 +// negation flags) are named after the typed getters or the directive. +// +// Syntax is the compiler's, byte for byte, except where its printer emits text +// its own parser rejects; the parsers here also accept that text, and the +// printers here then emit the form the compiler parses back to the same type: +// - atom.universal_copy with copy_bits 0 and an address space prints `0 b`; +// - atom.simt_multimem_* print a default field when a later one is not; +// - sm100.mma_bs(_sp) print `arch_promote` when `int_overflow` is present. +// +// Verifiers carry the compiler's rules on parameter values where they are +// known, with its messages; instruction legality (the MMA shape and type +// tables, multimem PTX forms, ldmatrix/stmatrix configurations) is left to the +// compiler. +// +// Left out: the hidden `atom` attribute of the mma atoms' typed get()s. It +// overrides the atom's layouts, is never printed, and so cannot survive text; +// every mma atom parsed from text has it unset, which is what these types +// model. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" + +class CuteNVGPU_CustomType + : CuteNVGPU_Type { + let hasCustomAssemblyFormat = 1; +} + +defvar CuteNVGPU_Ty = "::mlir::Type"; +defvar CuteNVGPU_Shape = "::mlir::cutlass_compiler::cute::ShapeAttr"; +defvar CuteNVGPU_Layout = "::mlir::cutlass_compiler::cute::LayoutAttr"; +defvar CuteNVGPU_LayoutTy = "::mlir::cutlass_compiler::cute::LayoutType"; +defvar CuteNVGPU_StrideTy = "::mlir::cutlass_compiler::cute::StrideType"; +defvar CuteNVGPU_Major = "::mlir::cutlass_compiler::cute::MajorMode"; +defvar CuteNVGPU_Space = "::mlir::cutlass_compiler::cute::AddressSpace"; + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + +def CuteNVGPU_SmemDescType : CuteNVGPU_Type<"SmemDesc", "smem_desc"> { + let summary = "UMMA/GMMA shared-memory descriptor"; + let description = "`!cute_nvgpu.smem_desc`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_SmemDescSM107Type + : CuteNVGPU_Type<"SmemDescSM107", "sm107.smem_desc"> { + let summary = "SM107 shared-memory descriptor"; + let description = "`!cute_nvgpu.sm107.smem_desc`"; + let assemblyFormat = ""; +} + +class CuteNVGPU_SmemDescCircular + : CuteNVGPU_Type { + let summary = "Circular shared-memory descriptor over a block layout in bytes"; + let description = [{ + The block layout is static, has three modes, a 128-byte mode 1 and is + mode-1-major compact, with at least two blocks in mode 2: + + `!cute_nvgpu.}] # typeMnemonic # [{<<"(8,128,2):(128,1,1024)">>` + }]; + let parameters = (ins CuteNVGPU_Layout:$block_layout_bytes); + let assemblyFormat = "`<` $block_layout_bytes `>`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_SmemDescCircularSM103Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM103", + "sm103.smem_desc_circular">; +def CuteNVGPU_SmemDescCircularSM107Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM107", + "sm107.smem_desc_circular">; + +def CuteNVGPU_SmemDescViewType + : CuteNVGPU_CustomType<"SmemDescView", "smem_desc_view"> { + let summary = "Shared-memory descriptor iterator with a layout"; + let description = [{ + The descriptor is a smem_desc, sm107.smem_desc or smem_desc_circular type; + the layout is a plain layout, as a string. + + `!cute_nvgpu.smem_desc_view` + }]; + let parameters = (ins CuteNVGPU_Ty:$desc, CuteNVGPU_Layout:$layout); + let genVerifyDecl = 1; +} + +def CuteNVGPU_TmaDescriptorTiledType + : CuteNVGPU_Type<"TmaDescriptorTiled", "tma_descriptor_tiled"> { + let summary = "Tiled TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_tiled`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_TmaDescriptorIm2ColType + : CuteNVGPU_Type<"TmaDescriptorIm2Col", "tma_descriptor_im2col"> { + let summary = "Im2col TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_im2col`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_WorkIdResponseType + : CuteNVGPU_Type<"WorkIdResponse", "workid_response"> { + let summary = "Cluster launch control work-id response"; + let description = "`!cute_nvgpu.workid_response`"; + let assemblyFormat = ""; +} + +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomSIMTSyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTSyncCopy", "atom.universal_copy"> { + let summary = "Universal (SIMT) copy atom"; + let description = [{ + `copy_bits` 0 means the element width and is kept as 0; the address + spaces default to generic. Each field needs the previous ones. + + `!cute_nvgpu.atom.universal_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter:$src_space, + DefaultValuedParameter:$dst_space + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSIMTAsyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy"> { + let summary = "cp.async copy atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_async_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "LoadCacheMode":$cache, + "int":$copy_bits + ); +} + +def CuteNVGPU_CopyAtomDsmemStoreType + : CuteNVGPU_CustomType<"CopyAtomDsmemStore", "atom.dsmem_store"> { + let summary = "Distributed shared-memory store atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.dsmem_store` + }]; + let parameters = (ins CuteNVGPU_Ty:$val_type, "int":$copy_bits); + let genVerifyDecl = 1; +} + +class CuteNVGPU_BulkCopy + : CuteNVGPU_CustomType { + let summary = "Bulk (TMA non-tensor) copy atom"; + let description = [{ + Space-separated groups, printed when not default (`copy_bits` 0). + + `!cute_nvgpu.}] # typeMnemonic # "<" # example # ">`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, DefaultValuedParameter<"int", "0">:$copy_bits), + !if(!empty(flag), (ins), + !dag(ins, [DefaultValuedParameter<"bool", "false">], [flag]))); +} + +def CuteNVGPU_CopyAtomBulkCopyG2SType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s", "mcast", + "f32 copy_bits = 128 mcast = true">; +def CuteNVGPU_CopyAtomBulkCopyS2GType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g", "mask", + "f32 copy_bits = 128 mask = true">; +def CuteNVGPU_CopyAtomBulkCopyS2SType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s", "", + "f32 copy_bits = 128">; + +// g2r, r2g, r2s, s2r: ``, space separated, +// fields in parameter order and printed when not default; copy_bits absent or +// 0 parses as the element width and is always printed. + +def CuteNVGPU_CopyAtomG2RType + : CuteNVGPU_CustomType<"CopyAtomG2R", "atom.g2r"> { + let summary = "Global-to-register load atom"; + let description = [{ + `!cute_nvgpu.atom.g2r`, + `!cute_nvgpu.atom.g2r` (invariant only with + the default memory features) + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"L2PrefetchSize", "L2PrefetchSize::NONE">:$l2_prefetch_size, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"LoadCacheMode", "LoadCacheMode::always">:$load_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space, + DefaultValuedParameter<"bool", "false">:$invariant + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomR2GType + : CuteNVGPU_CustomType<"CopyAtomR2G", "atom.r2g"> { + let summary = "Register-to-global store atom"; + let description = [{ + `!cute_nvgpu.atom.r2g` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"StoreCacheMode", "StoreCacheMode::write_back">:$store_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + +class CuteNVGPU_SharedCopy + : CuteNVGPU_CustomType { + let summary = what # " atom"; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # [{` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomR2SType + : CuteNVGPU_SharedCopy<"CopyAtomR2S", "atom.r2s", "Register-to-shared store">; +def CuteNVGPU_CopyAtomS2RType + : CuteNVGPU_SharedCopy<"CopyAtomS2R", "atom.s2r", "Shared-to-register load">; + +// Multimem atoms: positional `, ORDER, SCOPE>`; a trailing +// run of defaults is omitted (copy_bits 0, ADD, no accumulation precision, +// relaxed, sys). + +def CuteNVGPU_CopyAtomSIMTMultimemStType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st"> { + let summary = "multimem.st atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. + + `!cute_nvgpu.atom.simt_multimem_st` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemRedType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red"> { + let summary = "multimem.red atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_multimem_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemLdReduce", + "atom.simt_multimem_ld_reduce"> { + let summary = "multimem.ld_reduce atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. The accumulation + precision must be given to give a memory order or scope. + + `!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + OptionalParameter<"LdReduceAccPrecisionKindAttr">:$ld_reduce_acc_precision, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory copy atoms: `` +//===----------------------------------------------------------------------===// + +class CuteNVGPU_TmemCopy + : CuteNVGPU_CustomType { + let summary = what; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # "`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, "int":$num_dp, "int":$num_bit, + "unsigned":$num_rep), + !dag(ins, [OptionalParameter<"::mlir::UnitAttr">], [flag])); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100TmemLoadType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemLoad", "atom.tmem_load", + "tcgen05.ld atom", "pack16b">; +def CuteNVGPU_CopyAtomSM100TmemStoreType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemStore", "atom.tmem_store", + "tcgen05.st atom", "expand16b">; + +def CuteNVGPU_CopyAtomSM10xTmemLoadRedType + : CuteNVGPU_CustomType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red"> { + let summary = "tcgen05.ld.red atom"; + let description = [{ + `half_split_off` prints as an integer attribute without spaces around `=`. + + `!cute_nvgpu.atom.tmem_load_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$nan, + OptionalParameter<"::mlir::IntegerAttr">:$half_split_off + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM107TmemLoadSPCompressType + : CuteNVGPU_CustomType<"CopyAtomSM107TmemLoadSPCompress", + "atom.tmem_load_spcompress"> { + let summary = "SM107 tensor-memory load with sparse compression"; + let description = [{ + `!cute_nvgpu.atom.tmem_load_spcompress` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$red, + OptionalParameter<"::mlir::UnitAttr">:$nan + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100CopyS2TType + : CuteNVGPU_CustomType<"CopyAtomSM100CopyS2T", "atom.s2t_copy"> { + let summary = "tcgen05.cp (shared to tensor memory) atom"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.s2t_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100S2TCopyV2Type + : CuteNVGPU_CustomType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2"> { + let summary = "tcgen05.cp atom with the shared-memory major mode"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.sm100_s2t_copy_v2` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + CuteNVGPU_Major:$smem_major, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomLdsmType + : CuteNVGPU_CustomType<"CopyAtomLdsm", "atom.ldsm"> { + let summary = "ldmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "LdsmSzPattern":$sz_pattern, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +def CuteNVGPU_CopyAtomStsmType + : CuteNVGPU_CustomType<"CopyAtomStsm", "atom.stsm"> { + let summary = "stmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.stsm, num_matrices = 4, n>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +// Executable: comma-separated fields up to g_stride, then space-separated +// groups printed when set (booleans when true, sparsity when not 1). + +def CuteNVGPU_CopyAtomTmaLoadType + : CuteNVGPU_CustomType<"CopyAtomTmaLoad", "atom.tma_load"> { + let summary = "Executable TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <"(64,1):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaLoadMode":$mode, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomTmaStoreType + : CuteNVGPU_CustomType<"CopyAtomTmaStore", "atom.tma_store"> { + let summary = "Executable TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.tma_store tma_gbasis = <"(64,16):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaStoreMode":$mode, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomTmaReduceType + : CuteNVGPU_CustomType<"CopyAtomTmaReduce", "atom.tma_reduce"> { + let summary = "Executable TMA reduce atom"; + let description = [{ + `!cute_nvgpu.atom.tma_reduce tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "TmaStoreMode":$mode, + "ReductionKind":$kind, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaLoadType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load"> { + let summary = "Executable im2col TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <"(64,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaStoreType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store"> { + let summary = "Executable im2col TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +// Non-executable: `<[kind, ]T, copy_bits = N, tma_gbasis = <"L">[, tma_format = F]>`. +// An omitted format is the element type's (F16_RN, BF16_RN, F32_RN, TF32_RN, +// F64_RN, U4 for 4-bit, U8 up to 8 bits, U16, U32/S32, U64/S64); it is always +// printed, and empty when the element type has none. + +class CuteNVGPU_NonExecTma + : CuteNVGPU_CustomType { + let summary = "Non-executable " # what # " atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + defvar common = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_LayoutTy:$tma_gbasis, + "std::optional":$tma_format + ); + let parameters = !if(!empty(kindType), common, + !con((ins kindType:$kind), common)); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaLoad", + "atom.non_exec_tiled_tma_load", "tiled TMA load", "TiledTmaLoad", + "sm_90, bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaStore", + "atom.non_exec_tiled_tma_store", "tiled TMA store", "", + "bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaReduce", + "atom.non_exec_tiled_tma_reduce", "tiled TMA reduce", "ReductionKind", + "ADD, f32, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F32_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaLoad", + "atom.non_exec_im2col_tma_load", "im2col TMA load", "Im2ColTmaLoad", + "sm_90, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaStore", + "atom.non_exec_im2col_tma_store", "im2col TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DGather4TmaLoad", + "atom.non_exec_2d_gather4_tma_load", "2-D gather4 TMA load", + "GatherScatterTmaLoad", + "sm_100, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DScatter4TmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DScatter4TmaStore", + "atom.non_exec_2d_scatter4_tma_store", "2-D scatter4 TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(4,64):(1@0,1@1)\">, tma_format = F16_RN">; + +//===----------------------------------------------------------------------===// +// MMA atoms. The shape is `MxNxK` (a static rank-3 shape); `elem_type` lists +// A, B and C; `frag_kind` is the A fragment kind: `ss` (smem_desc), `rs` +// (rmem, sm90) or `ts` (tmem, sm100 and sm107). +//===----------------------------------------------------------------------===// + +class CuteNVGPU_MmaAtom + : CuteNVGPU_CustomType { + let summary = what # " MMA atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_UniversalFmaAtomType + : CuteNVGPU_MmaAtom<"UniversalFmaAtom", "atom.universal_fma", "Universal FMA", + "1x1x1, (f32, f32) -> f32 "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM80Type + : CuteNVGPU_MmaAtom<"MmaAtomSM80", "sm80.mma", "SM80 mma.sync", + "16x8x32, elem_type = (i8, i8, i32), int_overflow = "> { + let description = [{ + `binary_op` needs `int_overflow` before it. + + `!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"BinaryOpAttr">:$binary_op + ); +} + +def CuteNVGPU_MmaAtomSM80SparseType + : CuteNVGPU_MmaAtom<"MmaAtomSM80Sparse", "sm80.sparse_mma", + "SM80 sparse mma.sync", + "16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = "> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "SparseMetadataFormat":$sparse_metadata_format, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow + ); +} + +def CuteNVGPU_MmaAtomSM89Type + : CuteNVGPU_MmaAtom<"MmaAtomSM89", "sm89.mma", "SM89 fp8 mma.sync", + "16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM90Type + : CuteNVGPU_MmaAtom<"MmaAtomSM90", "sm90.mma", "SM90 wgmma", + "64x128x16, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = rs, a_neg, b_neg"> { + let description = [{ + The trailing `int_overflow`, `a_neg` and `b_neg` parse in any order and + print in this one. + + `!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "MmaFragKind":$a_frag_kind, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"::mlir::UnitAttr">:$a_neg, + OptionalParameter<"::mlir::UnitAttr">:$b_neg + ); +} + +def CuteNVGPU_MmaAtomSM120BlockScaledType + : CuteNVGPU_MmaAtom<"MmaAtomSM120BlockScaled", "SM120.mma_bs", + "SM120 block-scaled mma.sync", + "16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true"> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + "int":$vec_size, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + CuteNVGPU_Ty:$sf_type, + "bool":$use_sf_layout_TV + ); +} + +// UMMA (tcgen05.mma): `]>`. arch_promote (sm100 +// block-scaled only) defaults to sm_100 and is omitted then; the sm107 atoms +// require ab_collector_op. + +class CuteNVGPU_Umma + : CuteNVGPU_MmaAtom { + let parameters = !con( + (ins CuteNVGPU_Shape:$shape_mnk, "int":$num_cta, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type), + !if(sf, (ins CuteNVGPU_Ty:$sf_type), (ins)), + !if(!and(sp, !not(sf)), (ins CuteNVGPU_Ty:$e_type), (ins)), + !if(sp, (ins "SparseMetadataFormat":$sparse_metadata_format), (ins)), + (ins "MmaFragKind":$a_frag_kind), + !if(bs, (ins "int":$vec_size), (ins "int":$c_scale_exp)), + !if(!and(bs, !not(sm107)), + (ins DefaultValuedParameter<"Arch", "Arch::sm_100">:$arch_promote), (ins)), + !if(sm107, (ins "MmaCollectorOp":$a_collector_op, + "MmaCollectorOp":$b_collector_op), (ins)), + (ins OptionalParameter<"MMAIntOverflowAttr">:$int_overflow)); +} + +def CuteNVGPU_MmaAtomSM100UMMAType + : CuteNVGPU_Umma<"MmaAtomSM100UMMA", "sm100.mma", "SM100 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0", + 0, 0, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMASparse", "sm100.mma_sp", "SM100 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0", + 0, 1, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs", + "SM100 block-scaled UMMA", + "128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103", + 1, 0, 1, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaledSparse", "sm100.mma_bs_sp", + "SM100 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32", + 1, 1, 1, 0>; +def CuteNVGPU_MmaAtomSM107UMMAType + : CuteNVGPU_Umma<"MmaAtomSM107UMMA", "sm107.mma", "SM107 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)", + 0, 0, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMASparse", "sm107.mma_sp", "SM107 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)", + 0, 1, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaled", "sm107.mma_bs", + "SM107 block-scaled UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = ", + 1, 0, 1, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaledSparse", "sm107.mma_bs_sp", + "SM107 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)", + 1, 1, 1, 1>; + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD diff --git a/mlir/dialects/cute_nvgpu/capi.cc b/mlir/dialects/cute_nvgpu/capi.cc new file mode 100644 index 00000000..73d55bc3 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/capi.cc @@ -0,0 +1,4429 @@ +//===- capi.cc - C API for the cute_nvgpu dialect -------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) + +// Attributes + +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaledSparse(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM120BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); +} + +bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, + MlirType valueType, uint32_t operand, + bool deriveElemType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, false, valueTypeValue)) + return {nullptr}; + auto operandValue = + ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); + if (!operandValue) + return {nullptr}; + return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, + deriveElemType)); +} +MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValueType())); +} +uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getOperand()); +} +bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDeriveElemType(); +} + +bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, + MlirType aType, + MlirType eType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getAType())); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getEType())); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, + MlirType dataType, + MlirType storageType, + int ctaGroup, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type dataTypeValue; + if (!unwrapAs(ctx, dataType, false, dataTypeValue)) + return {nullptr}; + mlir::Type storageTypeValue; + if (!unwrapAs(ctx, storageType, false, storageTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemFrgAttr::get( + ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getDataType())); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getStorageType())); +} +int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, + MlirType sfType, int sfVecSize, + int ctaGroup, bool isSfa, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( + ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getSfType())); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getSfVecSize(); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getIsSfa(); +} +uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +// Types + +bool mlirTypeIsACuteNVGPUType(MlirType type) { + mlir::Type cppType = unwrap(type); + if (!cppType) + return false; + if (auto opaque = mlir::dyn_cast(cppType)) + return opaque.getDialectNamespace() == "cute_nvgpu"; + return cppType.getDialect().getNamespace() == "cute_nvgpu"; +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(MlirContext context, + MlirType valType, int copyBits, + bool mcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, valTypeValue, + copyBits, mcast)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(MlirContext context, + MlirType valType, int copyBits, + bool mask) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, valTypeValue, + copyBits, mask)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type) { + return llvm::cast(unwrap(type)) + .getMask(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(MlirContext context, + MlirType valType, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, valTypeValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomDsmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l2PrefetchSizeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeL2PrefetchSize( + l2PrefetchSize); + if (!l2PrefetchSizeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto loadCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode( + loadCacheMode); + if (!loadCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomG2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l2PrefetchSizeValue, *l1CacheEvictPriorityValue, *loadCacheModeValue, + *sharedSpaceValue, invariant)); +} +MlirType mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL2PrefetchSize()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getLoadCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} +bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type) { + return llvm::cast(unwrap(type)).getInvariant(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(MlirContext context, + MlirType valType, + int copyBits, int numCta, + MlirType gStride, bool mcast, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get( + ctx, valTypeValue, copyBits, numCta, gStrideValue, mcast, + tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +bool mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType gStride, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get( + ctx, valTypeValue, copyBits, gStrideValue, tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, + uint32_t szPattern, int numMatrices, + bool transpose) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + auto szPatternValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(szPattern); + if (!szPatternValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomLdsmType::get( + ctx, valTypeValue, modeValue, *szPatternValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMode())); +} +uint32_t mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSzPattern()); +} +int mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type) { + auto value = llvm::cast( + unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t l1CacheEvictPriority, + uint32_t storeCacheMode, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto storeCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeStoreCacheMode( + storeCacheMode); + if (!storeCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2GType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l1CacheEvictPriorityValue, *storeCacheModeValue, *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getStoreCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2SType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomS2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(MlirContext context, + MlirType valType, + uint32_t cache, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto cacheValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(cache); + if (!cacheValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get( + ctx, valTypeValue, *cacheValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getCache()); +} +int mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::LdReduceAccPrecisionKindAttr + ldReduceAccPrecisionValue; + if (!unwrapAs(ctx, ldReduceAccPrecision, true, ldReduceAccPrecisionValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get( + ctx, valTypeValue, copyBits, *reductionValue, ldReduceAccPrecisionValue, + *memOrderValue, *memScopeValue)); +} +MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getLdReduceAccPrecision())); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get( + ctx, valTypeValue, copyBits, *reductionValue, *memOrderValue, + *memScopeValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(MlirContext context, + MlirType valType, + int copyBits, + uint32_t memOrder, + uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get( + ctx, valTypeValue, copyBits, *memOrderValue, *memScopeValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(MlirContext context, + MlirType valType, + int copyBits, + uint32_t srcSpace, + uint32_t dstSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto srcSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(srcSpace); + if (!srcSpaceValue) + return {nullptr}; + auto dstSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(dstSpace); + if (!dstSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *srcSpaceValue, *dstSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSrcSpace()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getDstSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t broadcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *broadcastValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t smemMajor, + uint32_t broadcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto smemMajorValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(smemMajor); + if (!smemMajorValue) + return {nullptr}; + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *smemMajorValue, *broadcastValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSmemMajor()); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, uint32_t numRep, + bool pack16b) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + pack16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getPack16b()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, uint32_t numRep, + bool expand16b) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + expand16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getExpand16b()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool red, bool nan) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + red ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumBit(); +} +uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumRep(); +} +uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRed()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + ::mlir::IntegerAttr halfSplitOffValue; + if (!unwrapAs(ctx, halfSplitOff, true, halfSplitOffValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), halfSplitOffValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getHalfSplitOff())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, int numMatrices, + bool transpose) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomStsmType::get( + ctx, valTypeValue, modeValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMode())); +} +int mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(mode); + if (!modeValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, numCta, gStrideValue, mcast, + tmaGbasisValue, override, noFullyOobTile)); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)).getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet(MlirContext context, + MlirType valType, int copyBits, + uint32_t mode, uint32_t kind, + MlirType gStride, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) + return {nullptr}; + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaReduceType::get( + ctx, valTypeValue, copyBits, *modeValue, *kindValue, gStrideValue, + tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getKind()); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet(MlirContext context, + MlirType valType, int sparsity, + int copyBits, uint32_t mode, + MlirType gStride, + MlirType tmaGbasis, bool override, + bool noFullyOobTile) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, gStrideValue, + tmaGbasisValue, override, noFullyOobTile)); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); +} +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *archPromoteValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getArchPromote()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *archPromoteValue, + intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getArchPromote()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, vecSize, aTypeValue, bTypeValue, cTypeValue, sfTypeValue, + useSfLayoutTV)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +bool mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type) { + return llvm::cast(unwrap(type)) + .getUseSfLayout_TV(); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80SparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, + *sparseMetadataFormatValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType, + MlirAttribute intOverflow, + MlirAttribute binaryOp) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::BinaryOpAttr binaryOpValue; + if (!unwrapAs(ctx, binaryOp, true, binaryOpValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, intOverflowValue, + binaryOpValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBinaryOp())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM89Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM90Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, intOverflowValue, + aNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + bNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAFragKind()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getANeg()); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBNeg()); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM103Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); +} +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUSmemDescCircularSM107TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM107Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); +} +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescSM107Type::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirType desc, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type descValue; + if (!unwrapAs(ctx, desc, false, descValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + descValue, layoutValue)); +} +MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getDesc())); +} +MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorIm2ColType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorTiledType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet(MlirContext context, + MlirAttribute shapeMnk, + MlirType aType, MlirType bType, + MlirType cType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::UniversalFmaAtomType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); +} +MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} + +bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::WorkIdResponseType::get(ctx)); +} diff --git a/mlir/dialects/cute_nvgpu/capi.h b/mlir/dialects/cute_nvgpu/capi.h new file mode 100644 index 00000000..f1e4e999 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/capi.h @@ -0,0 +1,1481 @@ +//===- capi.h - C API for the cute_nvgpu dialect ------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CAPI_H +#define MLIR_DIALECTS_CUTE_NVGPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster +// Copy Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in +// CopyAtomDsmemStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in +// CopyAtomNonExec2DGather4TmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored +// in CopyAtomNonExec2DScatter4TmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in +// CopyAtomNonExecIm2ColTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in +// CopyAtomNonExecIm2ColTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in +// CopyAtomNonExecTiledTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in +// CopyAtomNonExecTiledTmaReduceType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in +// CopyAtomNonExecTiledTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy +// Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for +// SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom +// types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA +// Atom types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom +// types for SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom +// types for SM120 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for +// SM80. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different +// utccp instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in +// gather/scatter mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation +// precision kind. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.major`: Major mode for MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); + +// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, + uint32_t operand, bool deriveElemType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( + MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( + MlirContext context, MlirType aType, MlirType eType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( + MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, + uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( + MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, + bool isSfa, uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load +// instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); + +// Types + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet( + MlirContext context, MlirType valType, int copyBits, bool mcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type); + +// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet( + MlirContext context, MlirType valType, int copyBits, bool mask); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type); + +// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet( + MlirContext context, MlirType valType, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet( + MlirContext context, MlirType valType, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type); + +// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet( + MlirContext context, MlirType valType, int copyBits, int numCta, + MlirType gStride, bool mcast, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType gStride, + MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, + uint32_t szPattern, int numMatrices, bool transpose); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type); + +// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 +// TMA load atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D +// scatter4 TMA store atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2GTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l1CacheEvictPriority, uint32_t storeCacheMode, + uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2STypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomS2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet( + MlirContext context, MlirType valType, uint32_t cache, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t srcSpace, + uint32_t dstSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type); + +// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type); + +// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory +// major mode. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t smemMajor, uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type); + +// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, bool pack16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type); + +// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, bool expand16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type); + +// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse +// compression. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool red, bool nan); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type); + +// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type); + +// `!cute_nvgpu.atom.stsm`: stmatrix atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomStsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, int numMatrices, + bool transpose); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type); + +// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type); + +// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t mode, + uint32_t kind, MlirType gStride, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, MlirType gStride, MlirType tmaGbasis, bool override, + bool noFullyOobTile); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type); + +// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type); + +// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, MlirAttribute intOverflow, MlirAttribute binaryOp); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type); + +// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute shapeMnk, + MlirType aType, MlirType bType, MlirType cType); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type); + +// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type); + +// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet( + MlirContext context, MlirAttribute blockLayoutBytes); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type); + +// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM107TypeGet( + MlirContext context, MlirAttribute blockLayoutBytes); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type); + +// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context); + +// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context); + +// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a +// layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGet( + MlirContext context, MlirType desc, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type); + +// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context); + +// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context); + +// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type); + +// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CAPI_H diff --git a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig new file mode 100644 index 00000000..ac92ab5a --- /dev/null +++ b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig @@ -0,0 +1,5225 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const cute_ir = @import("mlir/dialects/cute_ir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvgpu"); +} + +/// `cute_nvgpu.grid_constant`: marks a kernel argument passed as a +/// `__grid_constant__` parameter. +pub fn gridConstant(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)); +} + +pub const NVGPUType = opaque { + const M = mlir.Methods(NVGPUType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteNVGPUType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; +}; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// NVGpu arch number like sm_103. +pub const Arch = enum(u32) { + sm_52 = 520, + sm_70 = 700, + sm_75 = 750, + sm_80 = 800, + sm_86 = 860, + sm_90 = 900, + sm_100 = 1000, + sm_103 = 1030, + sm_107 = 1070, +}; + +/// Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2S = enum(u32) { + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, +}; + +/// Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2G = enum(u32) { + byte_mask = 4, + cache_policy = 7, +}; + +/// Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2S = enum(u32) { + tma_bar = 1, + cta_rank = 5, +}; + +/// Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStore = enum(u32) { + mbar_ptr = 8, +}; + +/// Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoad = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStore = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduce = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoad = enum(u32) { + tma_descriptor_ptr = 0, + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduce = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStore = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100 = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + disable_output_lane = 6, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaled = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, + e = 5, +}; + +/// Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100Sparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + e = 5, +}; + +/// Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaled = enum(u32) { + sf_a = 0, + sf_b = 1, +}; + +/// Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80Sparse = enum(u32) { + e = 0, +}; + +/// Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90 = enum(u32) { + accum_c = 0, +}; + +/// Binary operation for single-bit MMA operations. +pub const BinaryOp = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Cute Cache Eviction Priority kind. +pub const CacheEvictionPriority = enum(u32) { + EVICT_NORMAL = 0, + EVICT_FIRST = 1, + EVICT_LAST = 2, + EVICT_UNCHANGED = 3, + NO_ALLOCATE = 4, +}; + +/// Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcast = enum(u32) { + none = 0, + lw_0213 = 1, + lw_0123 = 2, + x4 = 3, +}; + +/// The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoad = enum(u32) { + sm_100 = 0, + sm_100_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// CuTe L2 prefetch size options. +pub const L2PrefetchSize = enum(u32) { + NONE = 0, + RESERVED = 1, + SIZE_64B = 2, + SIZE_128B = 3, + SIZE_256B = 4, +}; + +/// multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKind = enum(u32) { + F16 = 0, + F32 = 1, +}; + +/// LDSM's sz pattern, describing the bit size. +pub const LdsmSzPattern = enum(u32) { + u16 = 0, + u4to8 = 1, + s4to8 = 2, + u2to4 = 3, + s2to4 = 4, + u4x16p64to8 = 5, + u6x16p32to8 = 6, + u8 = 7, +}; + +/// Cache modes for the load instructions. +pub const LoadCacheMode = enum(u32) { + always = 0, + global = 1, + streaming = 2, + last_use = 3, + none = 4, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + WEAK = 0, + RELAXED = 1, + ACQUIRE = 2, + RELEASE = 3, + ACQ_REL = 4, + SC = 5, + MMIO = 6, + CONSTANT = 7, + VOLATILE = 8, +}; + +/// Cute Memory Scope kind. +pub const MemScopeKind = enum(u32) { + CTA = 0, + CLUSTER = 1, + GPU = 2, + SYS = 3, +}; + +/// Enums for the mma collector op. +pub const MmaCollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// Enums for the mma frag type. +pub const MmaFragKind = enum(u32) { + smem_desc = 0, + sparse_smem_desc = 1, + tmem = 2, + tmem_ws = 3, + tmem_e = 4, + tmem_sf = 5, + rmem = 6, +}; + +/// Op for the TMASTORE instruction. +pub const ReductionKind = enum(u32) { + ADD = 0, + MIN = 1, + MAX = 2, + INC = 3, + DEC = 4, + AND = 5, + OR = 6, + XOR = 7, +}; + +/// CuTe Shared Space kind. +pub const SharedSpace = enum(u32) { + CTA = 0, + CLUSTER = 1, +}; + +/// Metadata format used in sparse MMA. +pub const SparseMetadataFormat = enum(u32) { + tid = 0, +}; + +/// Cache modes for the store instructions. +pub const StoreCacheMode = enum(u32) { + write_back = 0, + global = 1, + streaming = 2, + write_through = 3, + none = 4, +}; + +/// The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// Bits 74-71 of the TMA descriptor. +pub const TmaDataFormat = enum(u32) { + U8 = 0, + U16 = 1, + U32 = 2, + S32 = 3, + U64 = 4, + S64 = 5, + F16_RN = 6, + F32_RN = 7, + F32_FTZ_RN = 8, + F64_RN = 9, + BF16_RN = 10, + U4 = 11, + U4_UNPACK_U8 = 12, + U6_UNPACK_U8 = 13, + TF32_RN = 14, + TF32_FTZ_RN = 15, +}; + +/// Modes for the TMA load instruction. +pub const TmaLoadMode = enum(u32) { + tiled = 0, + im2col = 1, + w128 = 2, + w = 3, + gather4 = 4, +}; + +/// Modes for the TMA store instruction. +pub const TmaStoreMode = enum(u32) { + tiled = 0, + im2col = 1, + scatter4 = 2, +}; + +/// Modes for the Tmem allocation. +pub const TmemAllocMode = enum(u32) { + Interleaved = 0, + NonInterleaved = 1, + Duplicated = 2, + SfDuplicated_4x1 = 3, + SfDuplicated_2x2 = 4, +}; + +/// Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOp = enum(u32) { + max = 0, + maxabs = 1, + min = 2, + minabs = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyG2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_g2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + mcast: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mcast); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(self.ptr()); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyS2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2g"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + mask: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mask); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(self.ptr()); + } + pub fn mask(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyS2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. +pub const CopyAtomDsmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.dsmem_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. +pub const CopyAtomG2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.g2r"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l2PrefetchSize: L2PrefetchSize = .NONE, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + loadCacheMode: LoadCacheMode = .always, + sharedSpace: SharedSpace = .CTA, + invariant: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); + } + pub fn l2PrefetchSize(self: *const Self) L2PrefetchSize { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); + } + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); + } + pub fn loadCacheMode(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); + } + pub fn invariant(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. +pub const CopyAtomIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. +pub const CopyAtomIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. +pub const CopyAtomLdsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.ldsm"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + mode: *const mlir.Attribute, + szPattern: LdsmSzPattern, + numMatrices: c_int, + transpose: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn mode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn szPattern(self: *const Self) LdsmSzPattern { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); + } + pub fn numMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); + } + pub fn transpose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 TMA load atom. +pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_gather4_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: GatherScatterTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) GatherScatterTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D scatter4 TMA store atom. +pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_scatter4_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load atom. +pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: Im2ColTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) Im2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store atom. +pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load atom. +pub const CopyAtomNonExecTiledTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: TiledTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) TiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce atom. +pub const CopyAtomNonExecTiledTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: ReductionKind, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store atom. +pub const CopyAtomNonExecTiledTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. +pub const CopyAtomR2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.r2g"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + storeCacheMode: StoreCacheMode = .write_back, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); + } + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); + } + pub fn storeCacheMode(self: *const Self) StoreCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. +pub const CopyAtomR2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.r2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. +pub const CopyAtomS2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.s2r"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. +pub const CopyAtomSIMTAsyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_async_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + cache: LoadCacheMode, + copyBits: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn cache(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. +pub const CopyAtomSIMTMultimemLdReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_ld_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + reduction: ReductionKind = .ADD, + ldReduceAccPrecision: ?*const mlir.Attribute = null, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); + } + pub fn reduction(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); + } + pub fn ldReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. +pub const CopyAtomSIMTMultimemRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_red"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + reduction: ReductionKind = .ADD, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); + } + pub fn reduction(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. +pub const CopyAtomSIMTMultimemStType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_st"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. +pub const CopyAtomSIMTSyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + srcSpace: cute_ir.AddressSpace = .generic, + dstSpace: cute_ir.AddressSpace = .generic, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); + } + pub fn srcSpace(self: *const Self) cute_ir.AddressSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); + } + pub fn dstSpace(self: *const Self) cute_ir.AddressSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. +pub const CopyAtomSM100CopyS2TType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.s2t_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + broadcast: CopyS2TBroadcast = .none, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); + } + pub fn broadcast(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory major mode. +pub const CopyAtomSM100S2TCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.sm100_s2t_copy_v2"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + smemMajor: cute_ir.MajorMode, + broadcast: CopyS2TBroadcast = .none, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); + } + pub fn smemMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); + } + pub fn broadcast(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. +pub const CopyAtomSM100TmemLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + pack16b: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.pack16b); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(self.ptr()); + } + pub fn pack16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. +pub const CopyAtomSM100TmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + expand16b: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.expand16b); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(self.ptr()); + } + pub fn expand16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse compression. +pub const CopyAtomSM107TmemLoadSPCompressType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_spcompress"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + redOp: TmemLoadRedOp, + red: bool = false, + nan: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); + } + pub fn redOp(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); + } + pub fn red(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); + } + pub fn nan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +pub const CopyAtomSM10xTmemLoadRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_red"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + redOp: TmemLoadRedOp, + nan: bool = false, + halfSplitOff: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); + } + pub fn redOp(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); + } + pub fn nan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); + } + pub fn halfSplitOff(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.stsm`: stmatrix atom. +pub const CopyAtomStsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.stsm"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + mode: *const mlir.Attribute, + numMatrices: c_int, + transpose: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), args.numMatrices, args.transpose); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn mode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn numMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(self.ptr()); + } + pub fn transpose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. +pub const CopyAtomTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaLoadMode, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn sparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(self.ptr()); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaLoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn override(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(self.ptr()); + } + pub fn noFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. +pub const CopyAtomTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + mode: TmaStoreMode, + kind: ReductionKind, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); + } + pub fn kind(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. +pub const CopyAtomTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaStoreMode, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn sparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(self.ptr()); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn override(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(self.ptr()); + } + pub fn noFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. +pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn archPromote(self: *const Self) Arch { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. +pub const MmaAtomSM100UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn archPromote(self: *const Self) Arch { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. +pub const MmaAtomSM100UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. +pub const MmaAtomSM100UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +pub const MmaAtomSM107UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +pub const MmaAtomSM107UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. +pub const MmaAtomSM120BlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "SM120.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + vecSize: c_int, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + useSfLayoutTV: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.vecSize, args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), args.useSfLayoutTV); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(self.ptr()); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn useSfLayoutTV(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +pub const MmaAtomSM80SparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm80.sparse_mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +pub const MmaAtomSM80Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm80.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + intOverflow: ?*const mlir.Attribute = null, + binaryOp: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.binaryOp) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetCType(self.ptr()).ptr.?); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn binaryOp(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +pub const MmaAtomSM89Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm89.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetCType(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +pub const MmaAtomSM90Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm90.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + intOverflow: ?*const mlir.Attribute = null, + aNeg: bool = false, + bNeg: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn aNeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetANeg(self.ptr()); + } + pub fn bNeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. +pub const SmemDescCircularSM103Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm103.smem_desc_circular"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + blockLayoutBytes: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. +pub const SmemDescCircularSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc_circular"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + blockLayoutBytes: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM107TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +pub const SmemDescSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescSM107TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. +pub const SmemDescType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "smem_desc"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a layout. +pub const SmemDescViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "smem_desc_view"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + desc: *const mlir.Type, + layout: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.desc.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn desc(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetDesc(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. +pub const TmaDescriptorIm2ColType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_im2col"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. +pub const TmaDescriptorTiledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_tiled"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorTiledTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. +pub const UniversalFmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_fma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetCType(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. +pub const WorkIdResponseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "workid_response"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUWorkIdResponseTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulkg2s"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyG2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyG2S { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2GAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2g"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2G, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2G { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2s"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2S { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_dsmem_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldDsmemStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldDsmemStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_g2r"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldLoadGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldLoadGlobal { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_gather4_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DGather4TmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_scatter4_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DScatter4TmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_reduce"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_r2g"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldStoreGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldStoreGlobal { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmaload"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmareduce"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaReduce { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmastore"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100 { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaled { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaledSparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100Sparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm120_block_scaled"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM120BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM120BlockScaled { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm80_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM80Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM80Sparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm90"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM90, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM90 { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +pub const BinaryOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUBinaryOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bin_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BinaryOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BinaryOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUCopyS2TBroadcast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_s2t_broadcast_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CopyS2TBroadcast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUGatherScatterTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GatherScatterTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) GatherScatterTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "im2col_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Im2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Im2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_reduce_acc_precision_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdReduceAccPrecisionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdReduceAccPrecisionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +pub const LdsmSzPatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdsmSzPattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ldsm_sz_pattern"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdsmSzPattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdsmSzPattern { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +pub const LoadCacheModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULoadCacheMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAIntOverflow { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.major`: Major mode for MMA operations. +pub const MajorModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMajorMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "major"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +pub const MmaCollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaCollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_collector_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaCollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +pub const MmaFragKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaFragKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_frag_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaFragKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +pub const NotImplementedFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUNotImplementedFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "not_implemented_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUNotImplementedFrgAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_reduce_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +pub const RmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPURmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "rmem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: *const mlir.Type, + operand: cute_ir.MmaOperand, + deriveElemType: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valueType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); + } + pub fn operand(self: *const Self) cute_ir.MmaOperand { + return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); + } + pub fn deriveElemType(self: *const Self) bool { + return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); + } +}; + +/// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory fragment. +pub const SM100CircularSmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100CircularSmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.circular_smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +pub const SM100SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata fragment. +pub const SM100TmemEFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemEFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_e_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + aType: *const mlir.Type, + eType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemEFrgAttrGet(ctx.ptr(), args.aType.ptr(), args.eType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetAType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetEType(self.ptr()).ptr.?); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +pub const SM100TmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + dataType: *const mlir.Type, + storageType: *const mlir.Type, + ctaGroup: c_int, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn dataType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetDataType(self.ptr()).ptr.?); + } + pub fn storageType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(self.ptr()).ptr.?); + } + pub fn ctaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); + } + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor fragment. +pub const SM100TmemSfFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemSfFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_sf_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + sfType: *const mlir.Type, + sfVecSize: c_int, + ctaGroup: c_int, + isSfa: bool, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(self.ptr()).ptr.?); + } + pub fn sfVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(self.ptr()); + } + pub fn ctaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(self.ptr()); + } + pub fn isSfa(self: *const Self) bool { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); + } + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +pub const SM107SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM107SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm107.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +pub const SM90SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM90SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm90.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +pub const TmaDataFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaDataFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_data_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaDataFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaDataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +pub const TmaLoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaLoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaLoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaLoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +pub const TmaStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +pub const TmemAllocModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemAllocMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tmem_alloc_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemLoadRedOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tmem_load_red_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLoadRedOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cute_nvgpu.arch.alloc_rmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_rmem(ctx: *mlir.Context, input: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_rmem", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, input: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ + .named(ctx, "input", input), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.copy_s2t`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, dp: *const mlir.Attribute, bits: *const mlir.Attribute, cta: *const mlir.Attribute, broadcast: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.copy_s2t", .{ + .operands = .{ .flat = &.{ src, dst } }, + .attributes = &.{ + .named(ctx, "dp", dp), + .named(ctx, "bits", bits), + .named(ctx, "cta", cta), + .named(ctx, "broadcast", broadcast), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_load( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + coord: []const *const mlir.Value, + offsets: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_load", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + coord, + if (opts.multicast_mask) |value| &.{value} else &.{}, + offsets, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_reduce( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_reduce", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_store( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_store", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_load( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + pack_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (opts.pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_load", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_store( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + expand_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (opts.expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_store", .{ + .operands = .{ .flat = &.{ dst, src } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM107.tma_load_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_load_override( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_load_override", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (opts.upper_stride) |value| &.{value} else &.{}, + coord, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM107.tma_store_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_store_override( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_store_override", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (opts.upper_stride) |value| &.{value} else &.{}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_g2s( + ctx: *mlir.Context, + gmem_data_addr: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ + .operands = .{ .variadic = &.{ + &.{gmem_data_addr}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2g( + ctx: *mlir.Context, + smem_data_addr: *const mlir.Value, + gmem_data_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + byte_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ + .operands = .{ .variadic = &.{ + &.{smem_data_addr}, + &.{gmem_data_addr}, + if (opts.byte_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ + .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.make_warp_uniform`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_make_warp_uniform(ctx: *mlir.Context, val: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.make_warp_uniform", .{ + .operands = .{ .flat = &.{val} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, disable_output_lane: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma", .{ + .operands = .{ .flat = &.{ a, b, cd, idesc, accum, disable_output_lane } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, e_op: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, e_op, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, e: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, e, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM120_block_scaled( + ctx: *mlir.Context, + opA: []const *const mlir.Value, + opB: []const *const mlir.Value, + opC: []const *const mlir.Value, + sfA: *const mlir.Value, + sfB: *const mlir.Value, + res_types: []const *const mlir.Type, + shape_MNK: *const mlir.Attribute, + vec_size: *const mlir.Attribute, + a_type: *const mlir.Attribute, + b_type: *const mlir.Attribute, + sf_type: *const mlir.Attribute, + opts: struct { + byte_id_a: ?*const mlir.Value = null, + byte_id_b: ?*const mlir.Value = null, + thread_id_a: ?*const mlir.Attribute = null, + thread_id_b: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape_MNK", shape_MNK)); + attributes.appendAssumeCapacity(.named(ctx, "vec_size", vec_size)); + if (opts.thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); + if (opts.thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); + attributes.appendAssumeCapacity(.named(ctx, "a_type", a_type)); + attributes.appendAssumeCapacity(.named(ctx, "b_type", b_type)); + attributes.appendAssumeCapacity(.named(ctx, "sf_type", sf_type)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM120.block_scaled", .{ + .operands = .{ .variadic = &.{ + opA, + opB, + opC, + &.{sfA}, + &.{sfB}, + if (opts.byte_id_a) |value| &.{value} else &.{}, + if (opts.byte_id_b) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{ + res_types, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_prefetch_tma_desc(ctx: *mlir.Context, tma_descriptor_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{tma_descriptor_ptr} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.alloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_alloc_tmem( + ctx: *mlir.Context, + n_cols: *const mlir.Value, + dst_ptr: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.alloc_tmem", .{ + .operands = .{ .flat = &.{ n_cols, dst_ptr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.dealloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_dealloc_tmem( + ctx: *mlir.Context, + tmem_ptr: *const mlir.Value, + n_cols: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.dealloc_tmem", .{ + .operands = .{ .flat = &.{ tmem_ptr, n_cols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_relinquish_tmem_alloc_permit( + ctx: *mlir.Context, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.retrieve_tmem_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Value, tmem_ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", .{ + .operands = .{ .flat = &.{smem_ptr} }, + .results = .{ .flat = &.{tmem_ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_coord_tensor( + ctx: *mlir.Context, + atom: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + offset_b: ?*const mlir.Value = null, + layout_b: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{shape}, + if (opts.offset_b) |value| &.{value} else &.{}, + if (opts.layout_b) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_value", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_exec_tma( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + override: ?*const mlir.Attribute = null, + no_fully_oob_tile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); + if (opts.no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_gather4_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_coord_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_coord_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_scatter4_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_index_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_index_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_reduce( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_s2t_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_s2t_copy(ctx: *mlir.Context, copy_s2t_atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_s2t_copy", .{ + .operands = .{ .flat = &.{ copy_s2t_atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_tma_residue_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tma_residue_tensor(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, residue_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tma_residue_tensor", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor } }, + .results = .{ .flat = &.{residue_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_tmem_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tmem_copy(ctx: *mlir.Context, atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tmem_copy", .{ + .operands = .{ .flat = &.{ atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.set_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value_: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_sm107_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.tma_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_tma_partition(ctx: *mlir.Context, tma_atom: *const mlir.Value, cta_coord: *const mlir.Value, cta_layout: *const mlir.Value, smem_tensor: *const mlir.Value, target_tensors: []const *const mlir.Value, res_smem_tensor_type: *const mlir.Type, res_target_tensors_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.tma_partition", .{ + .operands = .{ .variadic = &.{ + &.{tma_atom}, + &.{cta_coord}, + &.{cta_layout}, + &.{smem_tensor}, + target_tensors, + } }, + .results = .{ .variadic = &.{ + &.{res_smem_tensor_type}, + res_target_tensors_types, + } }, + .location = location, + }); +} + +/// `cute_nvgpu.cast_tma_desc_to_integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast_tma_desc_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.cast_tma_desc_to_integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.copy_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.copy_tma_desc", .{ + .operands = .{ .flat = &.{ atom, tma_desc_ptr } }, + .location = location, + }); +} + +/// `cute_nvgpu.get_grid_constant_pointer`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_grid_constant_pointer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_grid_constant_pointer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.get_tma_desc_addr`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_tma_desc_addr(ctx: *mlir.Context, src: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_tma_desc_addr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.make_gmma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.make_gmma_smem_desc", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + result_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + &.{tma_desc_addr}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{atom} }, + .location = location, + }); +} + +/// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm103_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc_circular", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.update_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.update_tma_desc", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor, tma_desc_ptr } }, + .location = location, + }); +} diff --git a/mlir/dialects/cute_nvgpu/test.zig b/mlir/dialects/cute_nvgpu/test.zig new file mode 100644 index 00000000..3c7de6ce --- /dev/null +++ b/mlir/dialects/cute_nvgpu/test.zig @@ -0,0 +1,219 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cute_nvgpu.arch.alloc_rmem", + "cute_nvgpu.arch.alloc_smem", + "cute_nvgpu.arch.copy.SM100.copy_s2t", + "cute_nvgpu.arch.copy.SM100.tma_load", + "cute_nvgpu.arch.copy.SM100.tma_reduce", + "cute_nvgpu.arch.copy.SM100.tma_store", + "cute_nvgpu.arch.copy.SM100.tmem_load", + "cute_nvgpu.arch.copy.SM100.tmem_store", + "cute_nvgpu.arch.copy.SM107.tma_load_override", + "cute_nvgpu.arch.copy.SM107.tma_store_override", + "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", + "cute_nvgpu.arch.get_dyn_smem", + "cute_nvgpu.arch.get_dyn_smem_size", + "cute_nvgpu.arch.make_warp_uniform", + "cute_nvgpu.arch.mma.SM107.umma", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", + "cute_nvgpu.arch.mma.SM107.umma_sparse", + "cute_nvgpu.arch.mma.SM120.block_scaled", + "cute_nvgpu.arch.prefetch_tma_desc", + "cute_nvgpu.arch.sm100.alloc_tmem", + "cute_nvgpu.arch.sm100.dealloc_tmem", + "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", + "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", + "cute_nvgpu.atom.get_coord_tensor", + "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.get_value", + "cute_nvgpu.atom.make_exec_tma", + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", + "cute_nvgpu.atom.make_non_exec_im2col_tma_load", + "cute_nvgpu.atom.make_non_exec_im2col_tma_store", + "cute_nvgpu.atom.make_non_exec_tiled_tma_load", + "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", + "cute_nvgpu.atom.make_non_exec_tiled_tma_store", + "cute_nvgpu.atom.make_s2t_copy", + "cute_nvgpu.atom.make_tma_residue_tensor", + "cute_nvgpu.atom.make_tmem_copy", + "cute_nvgpu.atom.set_value", + "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.tma_partition", + "cute_nvgpu.cast_tma_desc_to_integer", + "cute_nvgpu.copy_tma_desc", + "cute_nvgpu.get_grid_constant_pointer", + "cute_nvgpu.get_tma_desc_addr", + "cute_nvgpu.make_gmma_smem_desc", + "cute_nvgpu.make_tma_desc_im2col", + "cute_nvgpu.make_tma_desc_im2col_at", + "cute_nvgpu.make_tma_desc_tiled", + "cute_nvgpu.make_tma_desc_tiled_at", + "cute_nvgpu.make_umma_smem_desc", + "cute_nvgpu.prefetch_tma_desc", + "cute_nvgpu.sm103.make_umma_smem_desc_circular", + "cute_nvgpu.sm107.make_umma_smem_desc", + "cute_nvgpu.sm107.make_umma_smem_desc_circular", + "cute_nvgpu.update_tma_desc", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_nvgpu); + inline for (comptime std.meta.declarations(cute_nvgpu)) |decl| { + const T = @field(cute_nvgpu, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "grid_constant is a unit attribute of the cute_nvgpu dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_nvgpu.gridConstant(ctx)})}); + try std.testing.expectEqualStrings("{cute_nvgpu.grid_constant}", w.buffered()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ cute_nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, + .{ cute_nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, + .{ cute_nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, + .{ cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, + .{ cute_nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, + .{ cute_nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, + .{ cute_nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, + .{ cute_nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, + .{ cute_nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, + .{ cute_nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, + .{ cute_nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, + .{ cute_nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, + .{ cute_nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, + .{ cute_nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, + .{ cute_nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, + .{ cute_nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, + .{ cute_nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, + .{ cute_nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, + .{ cute_nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, + .{ cute_nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, + .{ cute_nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, + .{ cute_nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, + .{ cute_nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, + .{ cute_nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, + .{ cute_nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, + .{ cute_nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, + .{ cute_nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, + .{ cute_nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, + .{ cute_nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, + .{ cute_nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, + .{ cute_nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, + .{ cute_nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, + .{ cute_nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, + .{ cute_nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, + .{ cute_nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, + .{ cute_nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, + .{ cute_nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, + .{ cute_nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_nvgpu}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } + try std.testing.expect((try cute_nvgpu.SmemDescType.get(ctx, .{})).type_().isA(cute_nvgpu.NVGPUType) != null); +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig deleted file mode 100644 index 3deac104..00000000 --- a/mlir/dialects/dialects.zig +++ /dev/null @@ -1,8 +0,0 @@ -const std = @import("std"); - -pub const stablehlo = @import("mlir/dialects/stablehlo"); -pub const func = @import("func.zig"); - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel new file mode 100644 index 00000000..87fe2405 --- /dev/null +++ b/mlir/dialects/fly/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], + tags = ["manual"], + deps = [ + "@flydsl//:FlyDialect", + "@flydsl//:TiledOpTraits", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "fly", + srcs = [ + "attributes.zig", + "ops.zig", + "types.zig", + ], + import_name = "mlir/dialects/fly", + main = "fly.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":fly", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + ], +) diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md new file mode 100644 index 00000000..9673870f --- /dev/null +++ b/mlir/dialects/fly/REGENERATING.md @@ -0,0 +1,79 @@ +# Regenerating the operation builders + +These are instructions for an agent. They describe what `ops.zig` and +`../fly_rocdl/ops.zig` must contain after a FlyDSL bump, where to derive them from, +and how to check the result. Scripts and intermediate files are not +committed; only the generated builders are. + +Paths below are relative to this directory. + +## Goal + +`ops.zig` binds every operation of the `fly` dialect and `../fly_rocdl/ops.zig` every +operation of the `fly_rocdl` dialect, one builder per operation, so that no +code builds a FlyDSL operation from its name. The attributes and types +(`attributes.zig`, `types.zig`, `../fly_rocdl/fly_rocdl.zig`, `capi.*`, `../fly_rocdl/capi.*`) are maintained by +hand and are not covered here. + +## Sources + +The FlyDSL commit is the one `third_party/flydsl/repo.bzl` pins, with its +patches applied: read the `.td` files from the fetched repository +(`$(bazel info output_base)/external/+non_module_deps+flydsl`), never from an +unpatched checkout. + +- `include/flydsl/Dialect/Fly/IR/FlyOps.td` for `ops.zig`. +- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `../fly_rocdl/ops.zig`. + +Dump their records with the LLVM the workspace builds: + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm-tblgen --dump-json \ + -I /include -I /external/+llvm+llvm-project/mlir/include \ + /include/flydsl/Dialect/Fly/IR/FlyOps.td -o FlyOps.json +``` + +Every record in `!instanceof.Op` whose `opDialect` names the dialect is an +operation. Do not regex the `.td` files. + +## What each builder must be + +Builder names, parameter order, `opts`, operand and result types, doc comments +and `operation_names` follow `mlir/dialects/README.md`. Specific to FlyDSL: + +- Builders are sorted by mnemonic. The doc comment gives the `summary` and + whether result types are inferred, explicit or absent. +- Unlike the README rule, parameter names are the ODS names in snake case. The + reserved locals `ctx`, `opts`, `location`, `make`, `operands`, `attributes` + and `present` also take a trailing underscore. +- Operations with `AttrSizedOperandSegments` pass every operand group through + `Operation.make`'s `.variadic` operands, which writes `operandSegmentSizes`; + others pass a flat list (an optional operand may only be last). +- Inferred operations set `.result_type_inference = true`. Inference comes + from the flattened trait list (trait lists such as `Pure` expand to their + members); an `OpInterface` record whose `cppInterfaceName` is + `InferTypeOpInterface` marks it. +- Attributes are `*const mlir.Attribute`, set under their ODS name. +- Builders call `make` from `fly.zig`, which verifies the operation and names + the operand types when creation fails. +- Each file starts with a header naming the FlyDSL commit and the `.td` it + was generated from. + +Regions, successors and `VariadicOfVariadic` operands have no use in either +dialect today; when one appears, extend the generator and this file. + +## Checks + +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/fly:test //dsl/fly:test`, which includes the + registration tests of the `operation_names` lists in both packages' + `test.zig`. +- Callers that no longer compile are the point of generating: an operation + whose signature changed in FlyDSL now fails at the call site. Fix them in + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, + and downstream llmd's `llmd/kernels/*/fly/`). +- Emit the IR of the fly kernels before and after (for instance + llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with + representative configs) and + diff it: a regeneration without an upstream change must be byte-identical. diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig new file mode 100644 index 00000000..3fc86ad3 --- /dev/null +++ b/mlir/dialects/fly/attributes.zig @@ -0,0 +1,223 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +/// `#fly.int`, the unified static/dynamic integer leaf. +pub const IntAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrStaticGet(ctx.ptr(), value).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrNoneGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.int_tuple`: a leaf, a basis element, or a nest of those. +pub const IntTupleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getStatic(ctx, value)).attribute() }); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getDynamic(ctx, width, divisibility)).attribute() }); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getNone(ctx)).attribute() }); + } + + pub fn getBasis(ctx: *mlir.Context, value: *const IntAttr, modes: []const i32) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrBasisGet(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const Self) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.layout`, a `shape:stride` pair. +pub const LayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + shape: *const IntTupleAttr, + stride: *const IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.tile`: a mode, or a `[a|b]` nest of modes. +pub const TileAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getModes(ctx: *mlir.Context, modes: []const *const mlir.Attribute) mlir.Error!*const Self { + const result = c.mlirFlyTileAttrModesGet(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `S`. +pub const SwizzleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + mask: i32, + base: i32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlySwizzleAttrGet(ctx.ptr(), args.mask, args.base, args.shift); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `align`, in bytes. +pub const AlignAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAlign; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + alignment: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyAlignAttrGet(ctx.ptr(), args.alignment).ptr orelse return error.InvalidMlir); + } +}; + +pub const AddressSpace = enum(i32) { generic = 0, global = 1, shared = 2, register = 3 }; + +/// `global`, `shared`, ... A pointer or memref may also carry a foreign +/// address-space attribute, such as `#fly_rocdl.buffer_desc`. +pub const AddressSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAddressSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + addressSpace: AddressSpace, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; + +pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} + +pub const GemmTraversalOrder = enum(i32) { + kmn = 0, + knm = 1, + mkn = 2, + mnk = 3, + nkm = 4, + nmk = 5, + kmn_serpentine = 6, + knm_serpentine = 7, + mkn_serpentine = 8, + mnk_serpentine = 9, + nkm_serpentine = 10, + nmk_serpentine = 11, +}; + +pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} diff --git a/mlir/dialects/fly/capi.cc b/mlir/dialects/fly/capi.cc new file mode 100644 index 00000000..1f4913bd --- /dev/null +++ b/mlir/dialects/fly/capi.cc @@ -0,0 +1,529 @@ +//===- capi.cc - C API for the fly dialect --------------------------------===// + +#include "capi.h" + +#include "TiledOpTraits.h" +#include "flydsl/Dialect/Fly/IR/FlyDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) + +using mlir::cast; +using mlir::isa; +using namespace mlir::fly; + +static mlir::MLIRContext *loadFly(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a +/// null test: every handle check here goes through these two helpers. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static T checked(Handle handle, mlir::MLIRContext *ctx) { + auto value = mlir::dyn_cast_if_present(unwrap(handle)); + if (isNull(value) || value.getContext() != ctx) + return T{}; + return value; +} + +#define FLY_ATTR_ISA(Name, Attr) \ + bool mlirAttributeIsAFly##Name(MlirAttribute attr) { \ + return mlir::isa_and_nonnull(unwrap(attr)); \ + } + +FLY_ATTR_ISA(Int, IntAttr) +FLY_ATTR_ISA(IntTuple, IntTupleAttr) +FLY_ATTR_ISA(Layout, LayoutAttr) +FLY_ATTR_ISA(Tile, TileAttr) +FLY_ATTR_ISA(Swizzle, SwizzleAttr) +FLY_ATTR_ISA(Align, AlignAttr) +FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) + +#undef FLY_ATTR_ISA + +#define FLY_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFly##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_TYPE_ISA(IntTuple, IntTupleType) +FLY_TYPE_ISA(Layout, LayoutType) +FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) +FLY_TYPE_ISA(Tile, TileType) +FLY_TYPE_ISA(Swizzle, SwizzleType) +FLY_TYPE_ISA(Pointer, PointerType) +FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) +FLY_TYPE_ISA(CoordTensor, CoordTensorType) +FLY_TYPE_ISA(CopyAtom, CopyAtomType) +FLY_TYPE_ISA(MmaAtom, MmaAtomType) +FLY_TYPE_ISA(TiledCopy, TiledCopyType) +FLY_TYPE_ISA(TiledMma, TiledMmaType) +FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) + +#undef FLY_TYPE_ISA + +MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getStatic(ctx, value))); +} + +MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, int32_t width, + int32_t divisibility) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + IntAttr::getDynamic(ctx, width, divisibility))); +} + +MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getNone(ctx))); +} + +MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(IntTupleAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyIntTupleAttrBasisGet(MlirContext context, + MlirAttribute value, intptr_t nModes, + const int32_t *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntAttr intValue = checked(value, ctx); + if (isNull(intValue) || (nModes > 0 && !modes)) + return {nullptr}; + return wrap(static_cast( + IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); +} + +MlirAttribute mlirFlyIntTupleAttrTupleGet(MlirContext context, + intptr_t nElements, + MlirAttribute const *elements) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nElements > 0 && !elements)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nElements); + for (intptr_t i = 0; i < nElements; ++i) { + IntTupleAttr element = checked(elements[i], ctx); + if (isNull(element)) + return {nullptr}; + values.push_back(element); + } + return wrap(static_cast( + IntTupleAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr shapeAttr = checked(shape, ctx); + IntTupleAttr strideAttr = checked(stride, ctx); + if (isNull(shapeAttr) || isNull(strideAttr)) + return {nullptr}; + return wrap(static_cast( + LayoutAttr::get(ctx, shapeAttr, strideAttr))); +} + +MlirAttribute mlirFlyTileAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(TileAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyTileAttrModesGet(MlirContext context, intptr_t nModes, + MlirAttribute const *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nModes > 0 && !modes)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nModes); + for (intptr_t i = 0; i < nModes; ++i) { + mlir::Attribute mode = unwrap(modes[i]); + if (isNull(mode) || mode.getContext() != ctx) + return {nullptr}; + values.push_back(mode); + } + return wrap(static_cast( + TileAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, int32_t mask, + int32_t base, int32_t shift) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(SwizzleAttr::get(ctx, mask, base, shift))); +} + +MlirAttribute mlirFlyAlignAttrGet(MlirContext context, int32_t alignment) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(AlignAttr::get(ctx, alignment))); +} + +MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext context, + int32_t addressSpace) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + AddressSpaceAttr::get(ctx, static_cast(addressSpace)))); +} + +MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, int32_t operand) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + MmaOperandAttr::get(ctx, static_cast(operand)))); +} + +MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext context, + int32_t order) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(GemmTraversalOrderAttr::get( + ctx, static_cast(order)))); +} + +MlirType mlirFlyIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(IntTupleType::get(value))); +} + +MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { + return cast(unwrap(type)).rank(); +} + +MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { + return wrap( + static_cast(cast(unwrap(type)).at(pos))); +} + +bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { + return cast(unwrap(type)).isLeaf(); +} + +bool mlirFlyIntTupleTypeIsStatic(MlirType type) { + return cast(unwrap(type)).isStatic(); +} + +bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { + IntTupleAttr attr = cast(unwrap(type)).getAttr(); + return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); +} + +bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafNone(); +} + +bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafBasis(); +} + +int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { + return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); +} + +MlirType mlirFlyLayoutTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + LayoutAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(LayoutType::get(value))); +} + +MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyLayoutTypeGetShape(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); +} + +MlirType mlirFlyLayoutTypeGetStride(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); +} + +MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyTileTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + TileAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(TileType::get(value))); +} + +MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlySwizzleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + SwizzleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(SwizzleType::get(value))); +} + +MlirType mlirFlyPointerTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + PointerType::get(ctx, element, space, align, sw))); +} + +MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAlignment())); +} + +MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getSwizzle())); +} + +MlirType mlirFlyMemRefTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, MlirAttribute layout, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + mlir::Attribute lay = unwrap(layout); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + if (isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + mlir::fly::MemRefType::get(ctx, element, space, lay, align, sw))); +} + +MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { + return wrap(cast(unwrap(type)).getLayout()); +} + +MlirType mlirFlyCoordTensorTypeGet(MlirContext context, MlirAttribute base, + MlirAttribute layout) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr baseAttr = checked(base, ctx); + mlir::Attribute lay = unwrap(layout); + if (isNull(baseAttr) || isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(CoordTensorType::get(ctx, baseAttr, lay))); +} + +static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { + while (auto composed = mlir::dyn_cast(layout)) + layout = composed.getOuter(); + return layout; +} + +MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { + mlir::Type ty = unwrap(type); + mlir::Attribute layout; + if (auto l = mlir::dyn_cast(ty)) { + layout = l.getAttr(); + } else if (auto c = mlir::dyn_cast(ty)) { + layout = c.getAttr(); + } else if (auto m = mlir::dyn_cast(ty)) { + layout = m.getLayout(); + } else { + layout = cast(ty).getLayout(); + } + return wrap(static_cast( + IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); +} + +MlirType mlirFlyCopyAtomTypeGet(MlirContext context, MlirType copyOp, + int32_t valBits) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(copyOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(CopyAtomType::get(ctx, op, valBits))); +} + +MlirType mlirFlyMmaAtomTypeGet(MlirContext context, MlirType mmaOp) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(mmaOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(MmaAtomType::get(ctx, op))); +} + +#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ + MlirType Name(MlirType type) { \ + return wrap(static_cast(LayoutType::get( \ + cast(cast(unwrap(type)).getter())))); \ + } + +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, + getThrValLayoutSrc) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, + getThrValLayoutDst) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, + getThrValLayoutRef) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, + getThrValLayoutA) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, + getThrValLayoutB) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, + getThrValLayoutC) + +#undef FLY_ATOM_LAYOUT + +MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { + return wrap(static_cast(IntTupleType::get( + cast(cast(unwrap(type)).getShapeMNK())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); +} + +static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { + auto ty = cast(unwrap(tiledMma)); + return wrap( + static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr(), operand)))); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::A); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::B); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::C); +} + +MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(CopyOpUniversalCopyType::get(ctx, bitSize))); +} diff --git a/mlir/dialects/fly/capi.h b/mlir/dialects/fly/capi.h new file mode 100644 index 00000000..50fb0f49 --- /dev/null +++ b/mlir/dialects/fly/capi.h @@ -0,0 +1,168 @@ +//===- capi.h - C API for the fly dialect -------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_CAPI_H +#define MLIR_DIALECTS_FLY_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, + int32_t width, + int32_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyIntTupleAttrBasisGet(MlirContext context, MlirAttribute value, + intptr_t nModes, const int32_t *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrTupleGet( + MlirContext context, intptr_t nElements, MlirAttribute const *elements); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrModesGet( + MlirContext context, intptr_t nModes, MlirAttribute const *modes); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, + int32_t mask, + int32_t base, + int32_t shift); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext context, + int32_t alignment); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyAddressSpaceAttrGet(MlirContext context, int32_t addressSpace); + +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, + int32_t operand); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyGemmTraversalOrderAttrGet(MlirContext context, int32_t order); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, + intptr_t pos); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); +MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyComposedLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext context, + MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext context, + MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, + MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet( + MlirContext context, MlirType elemTy, MlirAttribute addressSpace, + MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyMemRefTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext context, + MlirAttribute base, + MlirAttribute layout); + +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext context, + MlirType copyOp, + int32_t valBits); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext context, + MlirType mmaOp); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, int32_t bitSize); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_CAPI_H diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig new file mode 100644 index 00000000..15530388 --- /dev/null +++ b/mlir/dialects/fly/fly.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub const attributes = @import("attributes.zig"); +pub const ops = @import("ops.zig"); +pub const types = @import("types.zig"); + +pub const AddressSpace = attributes.AddressSpace; +pub const GemmTraversalOrder = attributes.GemmTraversalOrder; +pub const MmaOperand = attributes.MmaOperand; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly"); +} + +/// `Operation.make` that names the op and its operand types on failure: a +/// rejected `inferReturnTypes` otherwise panics with nothing to go on. +pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) *mlir.Operation { + return mlir.Operation.try_make(ctx, name, args) catch |err| { + var buf: [4096]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + if (args.operands) |operands| switch (operands) { + .flat => |vs| for (vs, 0..) |v, i| { + w.print("\n operand {d}: {f}", .{ i, v.type_() }) catch break; + }, + .variadic => |segments| for (segments) |seg| for (seg) |v| { + w.print("\n operand: {f}", .{v.type_()}) catch break; + }, + }; + if (args.results) |results| switch (results) { + .flat => |ts| for (ts, 0..) |t, i| { + w.print("\n result {d}: {f}", .{ i, t }) catch break; + }, + .variadic => {}, + }; + std.debug.panic("{s}: cannot create operation ({}){s}", .{ name, err, w.buffered() }); + }; +} + +pub const TypeKind = enum { + memref, + coord_tensor, + ptr, + int_tuple, + layout, + composed_layout, + tile, + swizzle, + copy_atom, + mma_atom, + tiled_copy, + tiled_mma, + other, +}; + +pub fn KindType(comptime kind: TypeKind) type { + return switch (kind) { + .memref => types.MemRefType, + .coord_tensor => types.CoordTensorType, + .ptr => types.PointerType, + .int_tuple => types.IntTupleType, + .layout => types.LayoutType, + .composed_layout => types.ComposedLayoutType, + .tile => types.TileType, + .swizzle => types.SwizzleType, + .copy_atom => types.CopyAtomType, + .mma_atom => types.MmaAtomType, + .tiled_copy => types.TiledCopyType, + .tiled_mma => types.TiledMmaType, + .other => @compileError("fly.KindType: `.other` is not a fly type"), + }; +} + +pub fn typeKind(ty: *const mlir.Type) TypeKind { + inline for (comptime std.meta.tags(TypeKind)) |kind| { + if (kind != .other and ty.isA(KindType(kind)) != null) return kind; + } + return .other; +} + +/// Panics if `ty` is not a `!fly.`. +pub fn expect(ty: *const mlir.Type, comptime kind: TypeKind) *const KindType(kind) { + return ty.isA(KindType(kind)) orelse + std.debug.panic("fly: `{f}` is not a !fly.{s}", .{ ty, @tagName(kind) }); +} + +/// The shape of the outermost plain layout of any layout-carrying type. +pub fn layoutLikeShape(ty: *const mlir.Type) *const types.IntTupleType { + if (ty.isA(types.LayoutType) == null and ty.isA(types.ComposedLayoutType) == null and + ty.isA(types.MemRefType) == null and ty.isA(types.CoordTensorType) == null) + { + std.debug.panic("fly: `{f}` carries no layout", .{ty}); + } + return @ptrCast(c.mlirFlyLayoutLikeTypeGetShape(ty.ptr()).ptr.?); +} + +pub fn elemType(ty: *const mlir.Type) *const mlir.Type { + if (ty.isA(types.MemRefType)) |m| return m.elemTy(); + if (ty.isA(types.PointerType)) |p| return p.elemTy(); + std.debug.panic("fly: `{f}` has no element type", .{ty}); +} + +pub fn addressSpace(ty: *const mlir.Type) *const mlir.Attribute { + if (ty.isA(types.MemRefType)) |m| return m.addressSpace(); + if (ty.isA(types.PointerType)) |p| return p.addressSpace(); + std.debug.panic("fly: `{f}` has no address space", .{ty}); +} + +/// Keeps the address space, alignment and swizzle. +pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir.Type) mlir.Error!*const types.PointerType { + const p = expect(ptr, .ptr); + return types.PointerType.get(ctx, .{ + .elemTy = elem, + .addressSpace = p.addressSpace(), + .alignment = p.alignment(), + .swizzle = p.swizzle(), + }); +} diff --git a/mlir/dialects/fly/ops.zig b/mlir/dialects/fly/ops.zig new file mode 100644 index 00000000..b9753136 --- /dev/null +++ b/mlir/dialects/fly/ops.zig @@ -0,0 +1,1152 @@ +//! `fly` operation builders, generated from FlyDSL's `include/flydsl/Dialect/Fly/IR/FlyOps.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("fly.zig").make; + +/// `fly.add_offset`. Result types are inferred. +pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.add_offset", .{ + .operands = .{ .flat = &.{ ptr, offset } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.append`. Result types are inferred. +pub fn append( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.append", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.apply_swizzle`. Result types are inferred. +pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzle: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.apply_swizzle", .{ + .operands = .{ .flat = &.{ ptr, swizzle } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.assume`. Result types are explicit. +pub fn assume(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.assume", .{ + .operands = .{ .flat = &.{ dst, src } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.atom.set_value`. Result types are inferred. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value: *const mlir.Value, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value } }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.blocked_product`. Result types are inferred. +pub fn blocked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.blocked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.ceil_div`. Result types are inferred. +pub fn ceil_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ceil_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coalesce`. Result types are inferred. +pub fn coalesce( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + pattern: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.coalesce", .{ + .operands = .{ .flat = if (opts.pattern) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.complement`. Result types are inferred. +pub fn complement( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + codomain_size: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.complement", .{ + .operands = .{ .flat = if (opts.codomain_size) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_inner`. Result types are inferred. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_offset`. Result types are inferred. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_outer`. Result types are inferred. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composition`. Result types are inferred. +pub fn composition(ctx: *mlir.Context, outer: *const mlir.Value, inner: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composition", .{ + .operands = .{ .flat = &.{ outer, inner } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coprofile`. Result types are inferred. +pub fn coprofile(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coprofile", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.copy`. No results. +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy", .{ + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call`. No results. +pub fn copy_atom_call( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy_atom_call", .{ + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call_ssa`. Result types are explicit. +pub fn copy_atom_call_ssa( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + dst: ?*const mlir.Value = null, + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + &.{src}, + if (opts.dst) |present| &.{present} else &.{}, + if (opts.pred) |present| &.{present} else &.{}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.coshape`. Result types are inferred. +pub fn coshape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coshape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.cosize`. Result types are inferred. +pub fn cosize(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.cosize", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.crd2idx`. Result types are inferred. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.decomposition`. Result types are inferred. +pub fn decomposition(ctx: *mlir.Context, tensor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.decomposition", .{ + .operands = .{ .flat = &.{tensor} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.dice`. Result types are inferred. +pub fn dice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.dice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.elem_less`. Result types are explicit. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.equal`. Result types are explicit. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.extract_aligned_pointer_as_index`. Extract the raw pointer from a fly.memref. Result types are explicit. +pub fn extract_aligned_pointer_as_index(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.flat_divide`. Result types are inferred. +pub fn flat_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.flat_product`. Result types are inferred. +pub fn flat_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.gemm`. Fully expand tiled GEMM with variadic A and B operand groups. No results. +pub fn gemm( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + d: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + opts: struct { + traversal_layout: ?*const mlir.Value = null, + traversal_order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); + return make(ctx, "fly.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + if (opts.traversal_layout) |present| &.{present} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.get`. Result types are inferred. +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get", .{ + .operands = .{ .flat = &.{input} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_1d_coord`. Result types are inferred. +pub fn get_1d_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_1d_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_copy_atom`. Extract the copy atom and its runtime state from a tiled copy. Result types are inferred. +pub fn get_copy_atom(ctx: *mlir.Context, tiled_copy: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_copy_atom", .{ + .operands = .{ .flat = &.{tiled_copy} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_dyn_shared`. Result types are inferred. +pub fn get_dyn_shared(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_dyn_shared", .{ + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_flat_coord`. Result types are inferred. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_iter`. Result types are inferred. +pub fn get_iter(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_iter", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_layout`. Result types are inferred. +pub fn get_layout(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_layout", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_leaves`. Result types are inferred. +pub fn get_leaves( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: struct { + dynamic_only: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); + return make(ctx, "fly.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_mma_atom`. Extract the MMA atom and its runtime state from a tiled MMA. Result types are inferred. +pub fn get_mma_atom(ctx: *mlir.Context, tiled_mma: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_mma_atom", .{ + .operands = .{ .flat = &.{tiled_mma} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_scalar`. Result types are inferred. +pub fn get_scalar(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_scalar", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_shape`. Result types are inferred. +pub fn get_shape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_shape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_stride`. Result types are inferred. +pub fn get_stride(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_stride", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.group`. Result types are inferred. +pub fn group(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.group", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.idx2crd`. Result types are inferred. +pub fn idx2crd(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.idx2crd", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_add`. Result types are inferred. +pub fn int_tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_div`. Result types are inferred. +pub fn int_tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mod`. Result types are inferred. +pub fn int_tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mul`. Result types are inferred. +pub fn int_tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product`. Result types are inferred. +pub fn int_tuple_product(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_each`. Result types are inferred. +pub fn int_tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_each", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_like`. Result types are inferred. +pub fn int_tuple_product_like(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_like", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_sub`. Result types are inferred. +pub fn int_tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.inttoptr`. Result types are explicit. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.left_inverse`. Result types are inferred. +pub fn left_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.left_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_divide`. Result types are inferred. +pub fn logical_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_product`. Result types are inferred. +pub fn logical_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_composed_layout`. Result types are inferred. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_coord`. Result types are explicit. +pub fn make_coord(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_coord", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_copy_atom`. Result types are explicit. +pub fn make_copy_atom(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, val_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_copy_atom", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "valBits", val_bits), + }, + .location = location, + }); +} + +/// `fly.make_fragment_layout_like`. Result types are inferred. +pub fn make_fragment_layout_like(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_fragment_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_fragment_like`. Result types are inferred. +pub fn make_fragment_like( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + dtype: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); + return make(ctx, "fly.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_identity_layout`. Result types are inferred. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_int_tuple`. Result types are explicit. +pub fn make_int_tuple(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_int_tuple", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_layout`. Result types are inferred. +pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout", .{ + .operands = .{ .flat = &.{ shape, stride } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_layout_like`. Result types are inferred. +pub fn make_layout_like(ctx: *mlir.Context, ref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout_like", .{ + .operands = .{ .flat = &.{ref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_mma_atom`. Result types are explicit. +pub fn make_mma_atom(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_mma_atom", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_ordered_layout`. Result types are inferred. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_ptr`. Result types are explicit. +pub fn make_ptr( + ctx: *mlir.Context, + args: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + dict_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); + return make(ctx, "fly.make_ptr", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.make_shape`. Result types are explicit. +pub fn make_shape(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_shape", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_stride`. Result types are explicit. +pub fn make_stride(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_stride", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_tiled_copy`. Result types are inferred. +pub fn make_tiled_copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, layout_thr_val: *const mlir.Value, tile_mn: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_tiled_copy", .{ + .operands = .{ .flat = &.{ copy_atom, layout_thr_val, tile_mn } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_tiled_mma`. Result types are inferred. +pub fn make_tiled_mma( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + atom_layout: *const mlir.Value, + opts: struct { + permutation: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.make_tiled_mma", .{ + .operands = .{ .flat = if (opts.permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_view`. Result types are inferred. +pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_view", .{ + .operands = .{ .flat = &.{ iter, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.alloca`. Result types are explicit. +pub fn memref_alloca(ctx: *mlir.Context, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.alloca", .{ + .operands = .{ .flat = &.{layout} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.memref.load`. Result types are inferred. +pub fn memref_load(ctx: *mlir.Context, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load", .{ + .operands = .{ .flat = &.{ memref, indices } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.load_vec`. Result types are inferred. +pub fn memref_load_vec(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load_vec", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.store`. No results. +pub fn memref_store(ctx: *mlir.Context, value: *const mlir.Value, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store", .{ + .operands = .{ .flat = &.{ value, memref, indices } }, + .location = location, + }); +} + +/// `fly.memref.store_vec`. No results. +pub fn memref_store_vec(ctx: *mlir.Context, vector: *const mlir.Value, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store_vec", .{ + .operands = .{ .flat = &.{ vector, memref } }, + .location = location, + }); +} + +/// `fly.mma.make_fragment`. Result types are inferred. +pub fn mma_make_fragment( + ctx: *mlir.Context, + tiled_mma: *const mlir.Value, + input: *const mlir.Value, + operand_id: *const mlir.Attribute, + opts: struct { + stages: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "operand_id", operand_id)); + if (opts.stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); + return make(ctx, "fly.mma.make_fragment", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.mma_atom_call`. No results. +pub fn mma_atom_call(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + } }, + .location = location, + }); +} + +/// `fly.mma_atom_call_ssa`. Result types are explicit. +pub fn mma_atom_call_ssa( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + d: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.mma_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + if (opts.d) |present| &.{present} else &.{}, + a, + b, + &.{c}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.prepend`. Result types are inferred. +pub fn prepend( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.prepend", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.print`. No results. +pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, format: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.print", .{ + .operands = .{ .flat = values }, + .attributes = &.{ + .named(ctx, "format", format), + }, + .location = location, + }); +} + +/// `fly.ptr.load`. Result types are explicit. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.ptr.store`. No results. +pub fn ptr_store(ctx: *mlir.Context, value: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.store", .{ + .operands = .{ .flat = &.{ value, ptr } }, + .location = location, + }); +} + +/// `fly.ptrtoint`. Result types are inferred. +pub fn ptrtoint(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptrtoint", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.raked_product`. Result types are inferred. +pub fn raked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.raked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.recast_iter`. Result types are explicit. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.recast_layout`. Result types are inferred. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.right_inverse`. Result types are inferred. +pub fn right_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.right_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.select`. Result types are inferred. +pub fn select(ctx: *mlir.Context, tuple: *const mlir.Value, indices: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.select", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "indices", indices), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.shape_div`. Result types are inferred. +pub fn shape_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.shape_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.size`. Result types are inferred. +pub fn size(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.size", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.slice`. Result types are inferred. +pub fn slice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.slice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.static`. Result types are explicit. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.take`. Result types are inferred. +pub fn take(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.take", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tile_to_shape`. Result types are inferred. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_dst`. Result types are inferred. +pub fn tiled_copy_partition_dst(ctx: *mlir.Context, tiled_copy: *const mlir.Value, dst: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_dst", .{ + .operands = .{ .flat = &.{ tiled_copy, dst, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_src`. Result types are inferred. +pub fn tiled_copy_partition_src(ctx: *mlir.Context, tiled_copy: *const mlir.Value, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_src", .{ + .operands = .{ .flat = &.{ tiled_copy, src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.retile`. Result types are inferred. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_divide`. Result types are inferred. +pub fn tiled_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition`. Result types are inferred. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition_shape`. Result types are inferred. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, shape: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, shape } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_product`. Result types are inferred. +pub fn tiled_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.to_llvm_ptr`. Result types are inferred. +pub fn to_llvm_ptr(ctx: *mlir.Context, ptr: *const mlir.Value, llvm_address_space: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.to_llvm_ptr", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = &.{ + .named(ctx, "llvm_address_space", llvm_address_space), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_divide`. Result types are inferred. +pub fn zipped_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_product`. Result types are inferred. +pub fn zipped_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/dialects/fly/test.zig b/mlir/dialects/fly/test.zig new file mode 100644 index 00000000..a30172ff --- /dev/null +++ b/mlir/dialects/fly/test.zig @@ -0,0 +1,378 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const fly = @import("mlir/dialects/fly"); +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); + +const attributes = fly.attributes; +const ops = fly.ops; +const types = fly.types; +const TypeKind = fly.TypeKind; +const typeKind = fly.typeKind; +const expect = fly.expect; +const layoutLikeShape = fly.layoutLikeShape; +const elemType = fly.elemType; +const ptrWithElem = fly.ptrWithElem; + +const operation_names: []const []const u8 = &.{ + "fly.add_offset", + "fly.append", + "fly.apply_swizzle", + "fly.assume", + "fly.atom.set_value", + "fly.blocked_product", + "fly.ceil_div", + "fly.coalesce", + "fly.complement", + "fly.composed_get_inner", + "fly.composed_get_offset", + "fly.composed_get_outer", + "fly.composition", + "fly.coprofile", + "fly.copy", + "fly.copy_atom_call", + "fly.copy_atom_call_ssa", + "fly.coshape", + "fly.cosize", + "fly.crd2idx", + "fly.decomposition", + "fly.dice", + "fly.elem_less", + "fly.equal", + "fly.extract_aligned_pointer_as_index", + "fly.flat_divide", + "fly.flat_product", + "fly.gemm", + "fly.get", + "fly.get_1d_coord", + "fly.get_copy_atom", + "fly.get_dyn_shared", + "fly.get_flat_coord", + "fly.get_iter", + "fly.get_layout", + "fly.get_leaves", + "fly.get_mma_atom", + "fly.get_scalar", + "fly.get_shape", + "fly.get_stride", + "fly.group", + "fly.idx2crd", + "fly.int_tuple_add", + "fly.int_tuple_div", + "fly.int_tuple_mod", + "fly.int_tuple_mul", + "fly.int_tuple_product", + "fly.int_tuple_product_each", + "fly.int_tuple_product_like", + "fly.int_tuple_sub", + "fly.inttoptr", + "fly.left_inverse", + "fly.logical_divide", + "fly.logical_product", + "fly.make_composed_layout", + "fly.make_coord", + "fly.make_copy_atom", + "fly.make_fragment_layout_like", + "fly.make_fragment_like", + "fly.make_identity_layout", + "fly.make_int_tuple", + "fly.make_layout", + "fly.make_layout_like", + "fly.make_mma_atom", + "fly.make_ordered_layout", + "fly.make_ptr", + "fly.make_shape", + "fly.make_stride", + "fly.make_tiled_copy", + "fly.make_tiled_mma", + "fly.make_view", + "fly.memref.alloca", + "fly.memref.load", + "fly.memref.load_vec", + "fly.memref.store", + "fly.memref.store_vec", + "fly.mma.make_fragment", + "fly.mma_atom_call", + "fly.mma_atom_call_ssa", + "fly.prepend", + "fly.print", + "fly.ptr.load", + "fly.ptr.store", + "fly.ptrtoint", + "fly.raked_product", + "fly.recast_iter", + "fly.recast_layout", + "fly.right_inverse", + "fly.select", + "fly.shape_div", + "fly.size", + "fly.slice", + "fly.static", + "fly.take", + "fly.tile_to_shape", + "fly.tiled_copy.partition_dst", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.retile", + "fly.tiled_divide", + "fly.tiled_mma.partition", + "fly.tiled_mma.partition_shape", + "fly.tiled_product", + "fly.to_llvm_ptr", + "fly.zipped_divide", + "fly.zipped_product", +}; + +test { + std.testing.refAllDecls(fly); + inline for (.{ types, attributes, ops, fly_rocdl, fly_rocdl.ops }) |module| { + std.testing.refAllDecls(module); + inline for (comptime std.meta.declarations(module)) |decl| { + const value = @field(module, decl); + if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + vector.registerDialect(registry); + memref.registerDialect(registry); + llvm.registerDialect(registry); + gpu.registerDialect(registry); + rocdl.registerDialect(registry); + ub.registerDialect(registry); + fly.registerDialect(registry); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "fly dialects register" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); + try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); + try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); + try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); + try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} + +test "fly types parse and print round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + const cases = [_][]const u8{ + "!fly.int_tuple<((2, 4), 8)>", + "!fly.int_tuple<((1, 4), (32, 64))>", + "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", + "!fly.memref", + "!fly.memref", + "!fly.ptr", + "!fly.ptr", + "!fly.ptr", + "!fly.memref", + "!fly_rocdl.cdna3.buffer_copy<128>", + "!fly_rocdl.cdna3.buffer_copy_lds<32>", + "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", + "!fly.copy_atom, 32>", + "!fly.tile<[8|64]>", + "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", + }; + for (cases) |src| { + const ty = mlir.Type.parse(ctx, src) catch |err| { + std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); + return err; + }; + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{ty}); + const ty2 = try mlir.Type.parse(ctx, w.buffered()); + try std.testing.expect(ty.eql(ty2)); + } +} + +test "types and attributes are built through the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + + const two = try attributes.IntTupleAttr.getStatic(ctx, 2); + const four = try attributes.IntTupleAttr.getStatic(ctx, 4); + const one = try attributes.IntTupleAttr.getStatic(ctx, 1); + const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); + const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); + const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); + try expectPrints("!fly.layout<(2,4):(4,1)>", layout); + try expectPrints("!fly.int_tuple<(2,4)>", layout.shape()); + try std.testing.expectEqual(@as(usize, 2), layout.shape().numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.shape().element(1).leaf()); + + const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); + const memref_type = try types.MemRefType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = global.attribute(), + .layout = layout.attr().attribute(), + }); + try expectPrints("!fly.memref", memref_type); + try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref_type.type_())); + + const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); + const ptr = try types.PointerType.get(ctx, .{ + .elemTy = .int(ctx, .i8), + .addressSpace = shared.attribute(), + .alignment = try .get(ctx, .{ .alignment = 16 }), + }); + try expectPrints("!fly.ptr>", ptr); + try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); + + const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ + (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), + (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), + }) }); + try expectPrints("!fly.tile<[8|64]>", tile); + + const copy_atom = try types.CopyAtomType.get(ctx, .{ + .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), + .valBits = 32, + }); + try expectPrints("!fly.copy_atom, 32>", copy_atom); + + const buffer_desc = try fly_rocdl.BufferDescAddressAttr.get(ctx); + try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = buffer_desc.attribute(), + })); +} + +test "fly enum attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + _ = try attributes.mmaOperandAttr(ctx, .a); + _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); +} + +test "inferred fly ops compute layout algebra" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = module.body(); + + // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. + const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); + const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); + const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); + const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); + const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); + + try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); + try std.testing.expect(module.operation().verify()); +} + +test "atom traits" { + const ctx = try testContext(); + defer ctx.deinit(); + + // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. + const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); + try expectPrints("!fly.int_tuple<(32,32,4)>", tm.tileSizeMNK()); + _ = tm.tiledThrValLayoutA(); + _ = tm.thrLayoutVMNK(); + + const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = .float(ctx, .f32), + .elemTyB = .float(ctx, .f32), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.mma_atom f32>>", atom); + try expectPrints("!fly.int_tuple<(16,16,4)>", atom.shapeMNK()); + try expectPrints("!fly.layout<64:1>", atom.thrLayout()); + + const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpGFX11WMMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .f16), + .elemTyB = .float(ctx, .f16), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.layout<32:1>", wmma.thrLayout()); + + const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); + _ = ca.thrValLayoutSrc(); +} + +test "type kinds" { + const ctx = try testContext(); + defer ctx.deinit(); + const cases = .{ + .{ "!fly.memref", TypeKind.memref }, + .{ "!fly.ptr", TypeKind.ptr }, + .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, + .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, + .{ "!fly.tile<[8|4]>", TypeKind.tile }, + .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, + .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, + }; + inline for (cases) |case| { + try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); + } + try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); +} + +test "structural readers" { + const ctx = try testContext(); + defer ctx.deinit(); + const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); + try std.testing.expectEqual(@as(usize, 2), it.numElements()); + try std.testing.expect(!it.isLeaf()); + try std.testing.expect(!it.isStatic()); + const inner = it.element(0); + try std.testing.expectEqual(@as(usize, 2), inner.numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.element(1).leaf()); + try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.element(1).leaf()); + + const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); + try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); + try expectPrints("bf16", elemType(m)); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/fly/types.zig b/mlir/dialects/fly/types.zig new file mode 100644 index 00000000..27966997 --- /dev/null +++ b/mlir/dialects/fly/types.zig @@ -0,0 +1,429 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +const attributes = @import("attributes.zig"); + +fn optAttr(value: anytype) c.MlirAttribute { + return if (value) |v| v.ptr() else .{ .ptr = null }; +} + +/// `!fly.int_tuple`. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.IntTupleAttr { + return @ptrCast(c.mlirFlyIntTupleTypeGetAttr(self.ptr()).ptr.?); + } + + /// 1 for a leaf. + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirFlyIntTupleTypeGetNumElements(self.ptr())); + } + + pub fn element(self: *const Self, pos: usize) *const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGetElement(self.ptr(), @intCast(pos)).ptr.?); + } + + pub fn isLeaf(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsLeaf(self.ptr()); + } + + pub fn isStatic(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsStatic(self.ptr()); + } + + pub const Leaf = union(enum) { static: i64, dynamic, none, basis }; + + /// Asserts the tuple is a leaf. + pub fn leaf(self: *const Self) Leaf { + if (c.mlirFlyIntTupleTypeIsStaticLeaf(self.ptr())) { + return .{ .static = c.mlirFlyIntTupleTypeGetStaticValue(self.ptr()) }; + } + if (c.mlirFlyIntTupleTypeIsNoneLeaf(self.ptr())) return .none; + if (c.mlirFlyIntTupleTypeIsBasisLeaf(self.ptr())) return .basis; + return .dynamic; + } +}; + +/// `!fly.layout`. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyLayoutTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.LayoutAttr { + return @ptrCast(c.mlirFlyLayoutTypeGetAttr(self.ptr()).ptr.?); + } + + pub fn shape(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetShape(self.ptr()).ptr.?); + } + + pub fn stride(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetStride(self.ptr()).ptr.?); + } +}; + +/// `!fly.composed_layout`. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.tile`. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.TileAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.TileAttr { + return @ptrCast(c.mlirFlyTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.swizzle`. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.SwizzleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlySwizzleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.ptr`. +pub const PointerType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyPointerTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn elemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyPointerTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn addressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyPointerTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn alignment(self: *const Self) *const attributes.AlignAttr { + return @ptrCast(c.mlirFlyPointerTypeGetAlignment(self.ptr()).ptr.?); + } + + pub fn swizzle(self: *const Self) *const attributes.SwizzleAttr { + return @ptrCast(c.mlirFlyPointerTypeGetSwizzle(self.ptr()).ptr.?); + } +}; + +/// `!fly.memref`. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + layout: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyMemRefTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), args.layout.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn elemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyMemRefTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn addressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!fly.coord_tensor`. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + base: *const attributes.IntTupleAttr, + layout: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCoordTensorTypeGet(ctx.ptr(), args.base.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.copy_atom`. +pub const CopyAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + copyOp: *const mlir.Type, + valBits: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyAtomTypeGet(ctx.ptr(), args.copyOp.ptr(), args.valBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn thrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn thrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn thrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutDst(self.ptr()).ptr.?); + } + + pub fn thrValLayoutRef(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `!fly.mma_atom`. +pub const MmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + mmaOp: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyMmaAtomTypeGet(ctx.ptr(), args.mmaOp.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn thrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn shapeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetShapeMNK(self.ptr()).ptr.?); + } + + pub fn thrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn thrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn thrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_copy`. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn tiledThrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_mma`. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn tileSizeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTileSizeMNK(self.ptr()).ptr.?); + } + + pub fn thrLayoutVMNK(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetThrLayoutVMNK(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.universal_copy`. +pub const CopyOpUniversalCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyOpUniversalCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyOpUniversalCopyTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; diff --git a/mlir/dialects/fly_rocdl/BUILD.bazel b/mlir/dialects/fly_rocdl/BUILD.bazel new file mode 100644 index 00000000..334e520b --- /dev/null +++ b/mlir/dialects/fly_rocdl/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], + tags = ["manual"], + deps = [ + "@flydsl//:FlyROCDLDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "fly_rocdl", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/fly_rocdl", + main = "fly_rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/fly", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":fly_rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/fly_rocdl/capi.cc b/mlir/dialects/fly_rocdl/capi.cc new file mode 100644 index 00000000..b055e993 --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.cc @@ -0,0 +1,107 @@ +//===- capi.cc - C API for the fly_rocdl dialect --------------------------===// + +#include "capi.h" + +#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, + mlir::fly_rocdl::FlyROCDLDialect) + +static mlir::MLIRContext *loadFlyROCDL(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// Null test for unwrapped handles, kept in one place as in fly's C API, where +/// `!attr` is value arithmetic on a fly IntAttr rather than a null test. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static MlirType mmaOpGet(MlirContext context, int32_t m, int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, + Flags... flags) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); + if (!ctx || isNull(a) || isNull(b) || isNull(acc)) + return {nullptr}; + if (a.getContext() != ctx || b.getContext() != ctx || acc.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(Type::get(ctx, m, n, k, a, b, acc, flags...))); +} + +#define FLY_ROCDL_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFlyROCDL##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopy, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopyLDS, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) +FLY_ROCDL_TYPE_ISA(MmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) + +#undef FLY_ROCDL_TYPE_ISA + +MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::BufferDescAddressAttr::get(ctx))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext context, + int32_t bitSize, + int32_t cacheModifier) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get( + ctx, bitSize, cacheModifier))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(ctx, bitSize))); +} + +MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc); +} + +MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} + +MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} diff --git a/mlir/dialects/fly_rocdl/capi.h b/mlir/dialects/fly_rocdl/capi.h new file mode 100644 index 00000000..0ee248bb --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.h @@ -0,0 +1,45 @@ +//===- capi.h - C API for the fly_rocdl dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_ROCDL_CAPI_H +#define MLIR_DIALECTS_FLY_ROCDL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet( + MlirContext context, int32_t bitSize, int32_t cacheModifier); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet( + MlirContext context, int32_t bitSize); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_ROCDL_CAPI_H diff --git a/mlir/dialects/fly_rocdl/fly_rocdl.zig b/mlir/dialects/fly_rocdl/fly_rocdl.zig new file mode 100644 index 00000000..7a91a925 --- /dev/null +++ b/mlir/dialects/fly_rocdl/fly_rocdl.zig @@ -0,0 +1,156 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly_rocdl"); +} + +/// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. +pub const BufferDescAddressAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn get(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyROCDLBufferDescAddressAttrGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy`. +pub const CopyOpCDNA3BufferCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + /// 0 = cached, 2 = non-temporal. + cacheModifier: i32 = 0, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(ctx.ptr(), args.bitSize, args.cacheModifier); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy_lds`. +pub const CopyOpCDNA3BufferCopyLDSType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.mfma`. +pub const MmaOpCDNA3MFMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpCDNA3MFMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx11.wmma`. +pub const MmaOpGFX11WMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX11WMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX11WMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx120x.wmma`. +pub const MmaOpGFX120XWMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX120XWMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `signA`/`signB`/`clamp` drive the integer WMMA intrinsics and must stay +/// false on the float paths. +pub const WmmaInitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + signA: bool = false, + signB: bool = false, + clamp: bool = false, +}; diff --git a/mlir/dialects/fly_rocdl/ops.zig b/mlir/dialects/fly_rocdl/ops.zig new file mode 100644 index 00000000..6cff2bf6 --- /dev/null +++ b/mlir/dialects/fly_rocdl/ops.zig @@ -0,0 +1,83 @@ +//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see ../fly/REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("mlir/dialects/fly").make; + +/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. +pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_load_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_store_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/dialects/fly_rocdl/test.zig b/mlir/dialects/fly_rocdl/test.zig new file mode 100644 index 00000000..bb85d6e5 --- /dev/null +++ b/mlir/dialects/fly_rocdl/test.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "fly_rocdl.get_buffer_rsrc", + "fly_rocdl.make_tiled_tdm_load_atom", + "fly_rocdl.make_tiled_tdm_store_atom", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(fly_rocdl); + inline for (comptime std.meta.declarations(fly_rocdl)) |decl| { + const T = @field(fly_rocdl, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/func/BUILD.bazel b/mlir/dialects/func/BUILD.bazel new file mode 100644 index 00000000..4524f2dd --- /dev/null +++ b/mlir/dialects/func/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:FuncExtensions", + ], +) + +zig_library( + name = "func", + import_name = "mlir/dialects/func", + main = "func.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":func", + "//mlir", + ], +) diff --git a/mlir/dialects/func/capi.cc b/mlir/dialects/func/capi.cc new file mode 100644 index 00000000..11a5e006 --- /dev/null +++ b/mlir/dialects/func/capi.cc @@ -0,0 +1,10 @@ +//===- capi.cc - C API for the func dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" + +void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { + mlir::func::registerAllExtensions(*unwrap(registry)); +} diff --git a/mlir/dialects/func/capi.h b/mlir/dialects/func/capi.h new file mode 100644 index 00000000..cd522b2d --- /dev/null +++ b/mlir/dialects/func/capi.h @@ -0,0 +1,20 @@ +//===- capi.h - C API for the func dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FUNC_CAPI_H +#define MLIR_DIALECTS_FUNC_CAPI_H + +#include "mlir-c/Dialect/Func.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_CAPI_EXPORTED void +mlirRegisterFuncExtensions(MlirDialectRegistry registry); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FUNC_CAPI_H diff --git a/mlir/dialects/func.zig b/mlir/dialects/func/func.zig similarity index 73% rename from mlir/dialects/func.zig rename to mlir/dialects/func/func.zig index b9f8b7f5..45c74674 100644 --- a/mlir/dialects/func.zig +++ b/mlir/dialects/func/func.zig @@ -1,8 +1,12 @@ -const std = @import("std"); +const c = @import("c"); const mlir = @import("mlir"); const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("func"); +} + const FuncOpArgs = struct { pub const Visibility = enum { public, @@ -17,18 +21,22 @@ const FuncOpArgs = struct { block: *mlir.Block, location: *const mlir.Location, no_inline: bool = false, - visibility: Visibility = .public, + visibility: ?Visibility = null, + verify: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { - var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; - var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 64) = .empty; - var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); + if (args.visibility) |v| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", .string(ctx, @tagName(v)))); + } attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), - .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), - .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( + .named(ctx, "function_type", .typeAttr(.function( ctx, args.args orelse args: { for (0..args.block.numArguments()) |i| { @@ -49,19 +57,23 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { }); if (args.args_attributes) |args_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); } if (args.results_attributes) |results_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); } if (args.no_inline) { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); + } + for (args.extra_attributes) |a| { + attr_tuples_buffer.appendAssumeCapacity(a); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, + .verify = args.verify, }); } @@ -70,7 +82,7 @@ pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Va .operands = .{ .variadic = &.{values} }, .results = .{ .flat = results }, .attributes = &.{ - .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), + .named(ctx, "callee", .flatSymbolRef(ctx, name)), }, .location = location, }); @@ -91,3 +103,7 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } + +pub fn registerExtensions(registry: *mlir.DialectRegistry) void { + c.mlirRegisterFuncExtensions(registry.ptr()); +} diff --git a/mlir/dialects/func/test.zig b/mlir/dialects/func/test.zig new file mode 100644 index 00000000..7a3b17f9 --- /dev/null +++ b/mlir/dialects/func/test.zig @@ -0,0 +1,41 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "func.call", + "func.func", + "func.return", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(func); + inline for (comptime std.meta.declarations(func)) |decl| { + const T = @field(func, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "func dialect and extensions register" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("func.func")); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/gpu/BUILD.bazel b/mlir/dialects/gpu/BUILD.bazel new file mode 100644 index 00000000..7264487a --- /dev/null +++ b/mlir/dialects/gpu/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:GPUDialect", + ], +) + +zig_library( + name = "gpu", + import_name = "mlir/dialects/gpu", + main = "gpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":gpu", + "//mlir", + ], +) diff --git a/mlir/dialects/gpu/capi.cc b/mlir/dialects/gpu/capi.cc new file mode 100644 index 00000000..2e215d58 --- /dev/null +++ b/mlir/dialects/gpu/capi.cc @@ -0,0 +1,52 @@ +//===- capi.cc - C API for the gpu dialect --------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) + +static mlir::MLIRContext *loadGPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional dimension = + mlir::gpu::symbolizeDimension(value); + if (!ctx || !dimension) + return {nullptr}; + return wrap(mlir::gpu::DimensionAttr::get(ctx, *dimension)); +} + +uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional mode = + mlir::gpu::symbolizeShuffleMode(value); + if (!ctx || !mode) + return {nullptr}; + return wrap(mlir::gpu::ShuffleModeAttr::get(ctx, *mode)); +} + +uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/gpu/capi.h b/mlir/dialects/gpu/capi.h new file mode 100644 index 00000000..670e23a9 --- /dev/null +++ b/mlir/dialects/gpu/capi.h @@ -0,0 +1,37 @@ +//===- capi.h - C API for the gpu dialect -------------------------*- C -*-===// +// +// C entry points for the `gpu` dialect that upstream's C API leaves out: its +// registration (outside `CAPIGPU`, which also brings the GPU passes) and the +// enum attributes kernels build. + +#ifndef MLIR_DIALECTS_GPU_CAPI_H +#define MLIR_DIALECTS_GPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUDimension(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr); + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_GPU_CAPI_H diff --git a/mlir/dialects/gpu/gpu.zig b/mlir/dialects/gpu/gpu.zig new file mode 100644 index 00000000..4b9df47f --- /dev/null +++ b/mlir/dialects/gpu/gpu.zig @@ -0,0 +1,169 @@ +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("gpu"); +} + +/// A dimension, either `x`, `y` or `z`. +pub const Dim = enum(u32) { + x = 0, + y = 1, + z = 2, +}; + +/// Indexing modes supported by `gpu.shuffle`. +pub const ShuffleMode = enum(u32) { + xor = 0, + down = 1, + up = 2, + idx = 3, +}; + +/// `#gpu`. +pub const DimensionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUDimension; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dim"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Dim, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Dim { + return @fromBackingInt(@intCast(c.mlirGPUDimensionAttrGetValue(self.ptr()))); + } +}; + +/// `#gpu`. +pub const ShuffleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUShuffleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shuffle_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShuffleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ShuffleMode { + return @fromBackingInt(@intCast(c.mlirGPUShuffleModeAttrGetValue(self.ptr()))); + } +}; + +/// `gpu.container_module`: marks a module whose `gpu.module`s hold the kernels +/// it launches. +pub fn containerModule(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.container_module", .unit(ctx)); +} + +/// `gpu.kernel`: marks a function as a kernel entry point. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.kernel", .unit(ctx)); +} + +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.module", .{ + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, name)), + }, + .location = location, + .verify = false, + }); +} + +pub const FuncArgs = struct { + name: []const u8, + args: []const *const mlir.Type, + block: *mlir.Block, + location: *const mlir.Location, + kernel: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, +}; + +/// `gpu.func @name(args) kernel { ... gpu.return }`. +pub fn func(ctx: *mlir.Context, args: FuncArgs) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "function_type", .typeAttr(.function(ctx, args.args, &.{}))), + }); + if (args.kernel) attrs.appendAssumeCapacity(kernel(ctx)); + attrs.appendSliceAssumeCapacity(args.extra_attributes); + return mlir.Operation.make(ctx, "gpu.func", .{ + .blocks = &.{args.block}, + .attributes = attrs.constSlice(), + .location = args.location, + .verify = false, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.return", .{ + .operands = .{ .flat = values }, + .location = location, + .verify = false, + }); +} + +fn idOp(ctx: *mlir.Context, comptime name: []const u8, dim: Dim, location: *const mlir.Location) *mlir.Operation { + const dimension = DimensionAttr.get(ctx, .{ .value = dim }) catch unreachable; + return mlir.Operation.make(ctx, name, .{ + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{.named(ctx, "dimension", dimension.attribute())}, + .location = location, + }); +} + +pub fn thread_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.thread_id", dim, location); +} + +pub fn block_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_id", dim, location); +} + +pub fn block_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_dim", dim, location); +} + +pub fn grid_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.grid_dim", dim, location); +} + +/// `gpu.shuffle`: `value` from lane `offset` (per `mode`) within `width` lanes, +/// and whether that lane was valid (i1). +pub fn shuffle(ctx: *mlir.Context, value: *const mlir.Value, offset: *const mlir.Value, width: *const mlir.Value, mode: ShuffleMode, location: *const mlir.Location) *mlir.Operation { + const mode_attr = ShuffleModeAttr.get(ctx, .{ .value = mode }) catch unreachable; + return mlir.Operation.make(ctx, "gpu.shuffle", .{ + .operands = .{ .flat = &.{ value, offset, width } }, + .results = .{ .flat = &.{ value.type_(), .int(ctx, .i1) } }, + .attributes = &.{.named(ctx, "mode", mode_attr.attribute())}, + .location = location, + }); +} + +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); +} diff --git a/mlir/dialects/gpu/test.zig b/mlir/dialects/gpu/test.zig new file mode 100644 index 00000000..d0f4a80d --- /dev/null +++ b/mlir/dialects/gpu/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const gpu = @import("mlir/dialects/gpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "gpu.barrier", + "gpu.block_dim", + "gpu.block_id", + "gpu.func", + "gpu.grid_dim", + "gpu.module", + "gpu.return", + "gpu.shuffle", + "gpu.thread_id", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(gpu); + inline for (comptime std.meta.declarations(gpu)) |decl| { + const T = @field(gpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "gpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(gpu.Dim).@"enum".field_names) |name| { + const value: gpu.Dim = @field(gpu.Dim, name); + const attr = try gpu.DimensionAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(gpu.DimensionAttr) != null); + try expectPrints("#gpu.dim<" ++ name ++ ">", attr); + } + inline for (@typeInfo(gpu.ShuffleMode).@"enum".field_names) |name| { + const value: gpu.ShuffleMode = @field(gpu.ShuffleMode, name); + const attr = try gpu.ShuffleModeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); + } +} + +test "container_module and kernel are unit attributes of the gpu dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ gpu.containerModule(ctx), gpu.kernel(ctx) })}); + try std.testing.expectEqualStrings("{gpu.container_module, gpu.kernel}", w.buffered()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + gpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/llvm/BUILD.bazel b/mlir/dialects/llvm/BUILD.bazel new file mode 100644 index 00000000..a03d215e --- /dev/null +++ b/mlir/dialects/llvm/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPILLVM", + "@llvm-project//mlir:LLVMDialect", + ], +) + +zig_library( + name = "llvm", + srcs = ["ops.zig"], + import_name = "mlir/dialects/llvm", + main = "llvm.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPILLVM", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":llvm", + "//mlir", + ], +) diff --git a/mlir/dialects/llvm/capi.cc b/mlir/dialects/llvm/capi.cc new file mode 100644 index 00000000..85b480e3 --- /dev/null +++ b/mlir/dialects/llvm/capi.cc @@ -0,0 +1,55 @@ +//===- capi.cc - C API for the llvm dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; + +static MLIRContext *loadLLVM(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +template static T optional(MlirAttribute attr) { + return llvm::dyn_cast_or_null(unwrap(attr)); +} + +bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopUnrollAttrGet(MlirContext context, + MlirAttribute disable, + MlirAttribute count, + MlirAttribute runtimeDisable, + MlirAttribute full) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopUnrollAttr::get(ctx, optional(disable), + optional(count), + optional(runtimeDisable), + optional(full), {}, {}, {})); +} + +bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopAnnotationAttrGet(MlirContext context, + MlirAttribute disableNonforced, + MlirAttribute unroll, + MlirAttribute mustProgress) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopAnnotationAttr::get( + ctx, optional(disableNonforced), {}, {}, + optional(unroll), {}, {}, {}, {}, {}, {}, + optional(mustProgress), {}, {}, {}, {})); +} diff --git a/mlir/dialects/llvm/capi.h b/mlir/dialects/llvm/capi.h new file mode 100644 index 00000000..04528ddc --- /dev/null +++ b/mlir/dialects/llvm/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the llvm dialect ------------------------*- C -*-===// +// +// C entry points for the `llvm` dialect's loop annotations, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_LLVM_CAPI_H +#define MLIR_DIALECTS_LLVM_CAPI_H + +#include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Null parameters are unset. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr); +/// `disable`, `runtimeDisable` and `full` are bool attributes, `count` an +/// integer attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopUnrollAttrGet( + MlirContext context, MlirAttribute disable, MlirAttribute count, + MlirAttribute runtimeDisable, MlirAttribute full); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr); +/// `disableNonforced` and `mustProgress` are bool attributes, `unroll` a +/// loop unroll attribute; the other parameters are unset. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopAnnotationAttrGet( + MlirContext context, MlirAttribute disableNonforced, MlirAttribute unroll, + MlirAttribute mustProgress); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_LLVM_CAPI_H diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig new file mode 100644 index 00000000..d1197317 --- /dev/null +++ b/mlir/dialects/llvm/llvm.zig @@ -0,0 +1,52 @@ +// Keep function definitions in ops.zig: LLVM reserves symbols starting with "llvm.". +const ops = @import("ops.zig"); + +pub const NVPTXAddressSpace = ops.NVPTXAddressSpace; +pub const pointerType = ops.pointerType; +pub const structType = ops.structType; +pub const identifiedStructType = ops.identifiedStructType; +pub const arrayType = ops.arrayType; +pub const LoopUnrollAttr = ops.LoopUnrollAttr; +pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; +pub const Linkage = ops.Linkage; +pub const linkageAttr = ops.linkageAttr; +pub const emitCInterface = ops.emitCInterface; +pub const linkage = ops.linkage; +pub const inlineHint = ops.inlineHint; +pub const noundef = ops.noundef; +pub const zeroext = ops.zeroext; +pub const signext = ops.signext; +pub const alignment = ops.alignment; +pub const dereferenceable = ops.dereferenceable; +pub const elementtype = ops.elementtype; +pub const AtomicBinOp = ops.AtomicBinOp; +pub const AtomicOrdering = ops.AtomicOrdering; +pub const inttoptr = ops.inttoptr; +pub const ptrtoint = ops.ptrtoint; +pub const bitcast = ops.bitcast; +pub const fmul = ops.fmul; +pub const extractvalue = ops.extractvalue; +pub const LoadOpts = ops.LoadOpts; +pub const load = ops.load; +pub const StoreOpts = ops.StoreOpts; +pub const store = ops.store; +pub const GEPIndex = ops.GEPIndex; +pub const getelementptr = ops.getelementptr; +pub const alloca = ops.alloca; +pub const atomicrmw = ops.atomicrmw; +pub const intr_assume = ops.intr_assume; +pub const intr_lifetime_start = ops.intr_lifetime_start; +pub const intr_lifetime_end = ops.intr_lifetime_end; +pub const extractelement = ops.extractelement; +pub const insertelement = ops.insertelement; +pub const shufflevector = ops.shufflevector; +pub const mlir_undef = ops.mlir_undef; +pub const mlir_poison = ops.mlir_poison; +pub const freeze = ops.freeze; +pub const BrOpts = ops.BrOpts; +pub const br = ops.br; +pub const unreachable_ = ops.unreachable_; +pub const call_intrinsic = ops.call_intrinsic; +pub const InlineAsmOpts = ops.InlineAsmOpts; +pub const inline_asm = ops.inline_asm; +pub const registerDialect = ops.registerDialect; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig new file mode 100644 index 00000000..d1fb5771 --- /dev/null +++ b/mlir/dialects/llvm/ops.zig @@ -0,0 +1,567 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("llvm"); +} + +// ============================================================================= +// Types +// ============================================================================= + +/// Address spaces of `!llvm.ptr` on NVPTX. +pub const NVPTXAddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + /// Tensor memory (SM100 TMEM). + tensor = 6, + /// Shared memory of any CTA of the cluster (`shared::cluster`). + shared_cluster = 7, +}; + +/// `!llvm.ptr`, an opaque pointer. Loads the LLVM dialect into `ctx`. +pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@backingInt(space))).ptr); +} + +/// `!llvm.struct<(fields...)>`, a literal struct. +pub fn structType(ctx: *mlir.Context, fields: []const *const mlir.Type) *const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMStructTypeLiteralGet(ctx.ptr(), @intCast(fields.len), @ptrCast(fields), false).ptr); +} + +/// `!llvm.struct<"name", (fields...)>`, an identified struct; null if `name` +/// already names a struct of other fields. +pub fn identifiedStructType(ctx: *mlir.Context, name: []const u8, fields: []const *const mlir.Type) ?*const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + const t = c.mlirLLVMStructTypeIdentifiedGet(ctx.ptr(), mlir.stringRef(name)); + if (c.mlirLLVMStructTypeSetBody(t, @intCast(fields.len), @ptrCast(fields), false).value == 0) return null; + return @ptrCast(t.ptr); +} + +/// `!llvm.array`. +pub fn arrayType(element_type: *const mlir.Type, count: u32) *const mlir.Type { + return @ptrCast(c.mlirLLVMArrayTypeGet(element_type.ptr(), count).ptr); +} + +// ============================================================================= +// Attributes +// ============================================================================= + +fn optionalBool(ctx: *mlir.Context, value: ?bool) c.MlirAttribute { + return if (value) |v| mlir.Attribute.boolean(ctx, v).ptr() else .{ .ptr = null }; +} + +/// `#llvm.loop_unroll`: the `llvm.loop.unroll.*` loop metadata. +pub const LoopUnrollAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopUnroll; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disable: ?bool = null, + count: ?u32 = null, + runtimeDisable: ?bool = null, + full: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const count: c.MlirAttribute = if (args.count) |n| mlir.Attribute.int(ctx, .i32, n).ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopUnrollAttrGet(ctx.ptr(), optionalBool(ctx, args.disable), count, optionalBool(ctx, args.runtimeDisable), optionalBool(ctx, args.full)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// A global's linkage, as `#llvm.linkage` spells it. +pub const Linkage = enum(c_uint) { + external = c.MlirLLVMLinkageExternal, + available_externally = c.MlirLLVMLinkageAvailableExternally, + linkonce = c.MlirLLVMLinkageLinkonce, + linkonce_odr = c.MlirLLVMLinkageLinkonceODR, + weak = c.MlirLLVMLinkageWeak, + weak_odr = c.MlirLLVMLinkageWeakODR, + appending = c.MlirLLVMLinkageAppending, + internal = c.MlirLLVMLinkageInternal, + private = c.MlirLLVMLinkagePrivate, + extern_weak = c.MlirLLVMLinkageExternWeak, + common = c.MlirLLVMLinkageCommon, +}; + +/// `#llvm.linkage`; `llvm.linkage` on a `func.func` sets its +/// linkage once lowered to LLVM. Loads the LLVM dialect into `ctx`, which +/// must have it registered. +pub fn linkageAttr(ctx: *mlir.Context, value: Linkage) *const mlir.Attribute { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(value)).ptr); +} + +/// `llvm.emit_c_interface`: has the lowering of a `func.func` emit a +/// `_mlir_ciface_` wrapper that takes its memrefs by pointer. +pub fn emitCInterface(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.emit_c_interface", .unit(ctx)); +} + +/// `llvm.linkage = #llvm.linkage` on a `func.func`. +pub fn linkage(ctx: *mlir.Context, value: Linkage) mlir.NamedAttribute { + return .named(ctx, "llvm.linkage", linkageAttr(ctx, value)); +} + +/// `llvm.inline_hint` on a `func.func`: `inlinehint` once lowered. +pub fn inlineHint(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.inline_hint", .unit(ctx)); +} + +/// `llvm.noundef` on an argument: never undef or poison. +pub fn noundef(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.noundef", .unit(ctx)); +} + +/// `llvm.zeroext` on an argument: a narrow integer passed zero-extended. +pub fn zeroext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.zeroext", .unit(ctx)); +} + +/// `llvm.signext` on an argument: a narrow integer passed sign-extended. +pub fn signext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.signext", .unit(ctx)); +} + +/// `llvm.align = bytes` on a pointer argument. +pub fn alignment(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.align", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.dereferenceable = bytes` on a pointer argument: that many bytes are +/// readable. +pub fn dereferenceable(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.dereferenceable", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.elementtype = type` on a pointer argument: its pointee, which +/// typed-pointer IR prints. +pub fn elementtype(ctx: *mlir.Context, pointee: *const mlir.Type) mlir.NamedAttribute { + return .named(ctx, "llvm.elementtype", .typeAttr(pointee)); +} + +/// The operation of `llvm.atomicrmw`, as the LLVM dialect numbers it. +pub const AtomicBinOp = enum(i64) { + xchg = 0, + add = 1, + sub = 2, + _and = 3, + nand = 4, + _or = 5, + _xor = 6, + max = 7, + min = 8, + umax = 9, + umin = 10, + fadd = 11, + fsub = 12, + fmax = 13, + fmin = 14, + uinc_wrap = 15, + udec_wrap = 16, + usub_cond = 17, + usub_sat = 18, + fmaximum = 19, + fminimum = 20, + fmaximumnum = 21, + fminimumnum = 22, +}; + +/// The memory ordering of an atomic operation, as the LLVM dialect numbers it. +pub const AtomicOrdering = enum(i64) { + not_atomic = 0, + unordered = 1, + monotonic = 2, + acquire = 4, + release = 5, + acq_rel = 6, + seq_cst = 7, +}; + +/// `#llvm.loop_annotation`: the `llvm.loop` metadata of a loop. +pub const LoopAnnotationAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopAnnotation; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disableNonforced: ?bool = null, + unroll: ?*const LoopUnrollAttr = null, + mustProgress: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const unroll: c.MlirAttribute = if (args.unroll) |u| u.ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopAnnotationAttrGet(ctx.ptr(), optionalBool(ctx, args.disableNonforced), unroll, optionalBool(ctx, args.mustProgress)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +// ============================================================================= +// Casts and arithmetic +// ============================================================================= + +/// llvm.inttoptr — an integer address as a pointer of type `result_type`. +pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.inttoptr", .{ + .operands = .{ .flat = &.{address} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.ptrtoint — the integer address of an LLVM pointer. +pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). +pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.bitcast", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.fmul — floating-point (or floating-point vector) product of type `result_type`. +pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.fmul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Aggregates +// ============================================================================= + +/// llvm.extractvalue — the element of type `result_type` at `position` in the +/// struct or array `container`. +pub fn extractvalue(ctx: *mlir.Context, container: *const mlir.Value, result_type: *const mlir.Type, position: []const i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{container} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "position", .denseArray(ctx, .i64, position))}, + .location = location, + }); +} + +// ============================================================================= +// Memory +// ============================================================================= + +pub const LoadOpts = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + /// The memory is not written while the kernel runs (`!invariant.load`; on NVPTX + /// global memory this selects the non-coherent `ld.global.nc`). + invariant: bool = false, +}; + +/// llvm.load — a value of type `result_type` through `pointer`. +pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + } + if (opts.invariant) { + attrs.appendAssumeCapacity(.named(ctx, "invariant", .unit(ctx))); + } + return mlir.Operation.make(ctx, "llvm.load", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StoreOpts = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, +}; + +/// llvm.store — `value` through `pointer`. +pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + } + return mlir.Operation.make(ctx, "llvm.store", .{ + .operands = .{ .flat = &.{ value, pointer } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// An index of `getelementptr`: a constant, or an SSA integer. +pub const GEPIndex = union(enum) { + constant: i32, + value: *const mlir.Value, +}; + +/// llvm.getelementptr — `base` offset by `indices` into `elem_type`, with +/// `base`'s pointer type. +pub fn getelementptr( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const GEPIndex, + elem_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const dynamic_index = std.math.minInt(i32); + var operands: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + var raw: stdx.BoundedArray(i32, 16) = .empty; + operands.appendAssumeCapacity(base); + for (indices) |index| switch (index) { + .constant => |c_| raw.appendAssumeCapacity(c_), + .value => |v| { + raw.appendAssumeCapacity(dynamic_index); + operands.appendAssumeCapacity(v); + }, + }; + return mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{base.type_()} }, + .attributes = &.{ + .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, raw.constSlice())), + .named(ctx, "elem_type", .typeAttr(elem_type)), + }, + .location = location, + }); +} + +/// llvm.alloca — stack storage for `array_size` elements of `elem_type`, as a +/// pointer of type `result_type`. +pub fn alloca( + ctx: *mlir.Context, + array_size: *const mlir.Value, + result_type: *const mlir.Type, + elem_type: *const mlir.Type, + opts: struct { + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "elem_type", .typeAttr(elem_type))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + return mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{array_size} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// llvm.atomicrmw — combine `val` into the value at `ptr` with `bin_op`, +/// atomically; the result is the previous value, of `val`'s type. +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + val: *const mlir.Value, + bin_op: AtomicBinOp, + ordering: AtomicOrdering, + opts: struct { + /// Synchronization scope, e.g. `"agent"` (none: the whole system). + syncscope: ?[]const u8 = null, + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "bin_op", .int(ctx, .i64, @backingInt(bin_op))), + .named(ctx, "ordering", .int(ctx, .i64, @backingInt(ordering))), + }); + if (opts.syncscope) |syncscope| attrs.appendAssumeCapacity(.named(ctx, "syncscope", .string(ctx, syncscope))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + return mlir.Operation.make(ctx, "llvm.atomicrmw", .{ + .operands = .{ .flat = &.{ ptr, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Intrinsics and inline assembly +// ============================================================================= + +/// llvm.intr.assume — let the optimizer assume that `cond` (i1) holds. +pub fn intr_assume(ctx: *mlir.Context, cond: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.assume", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + .named(ctx, "op_bundle_tags", .array(ctx, &.{})), + }, + .location = location, + }); +} + +pub fn intr_lifetime_start(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.start", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn intr_lifetime_end(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.end", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn extractelement(ctx: *mlir.Context, vector: *const mlir.Value, position: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractelement", .{ + .operands = .{ .flat = &.{ vector, position } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn insertelement(ctx: *mlir.Context, vector: *const mlir.Value, value: *const mlir.Value, position: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.insertelement", .{ + .operands = .{ .flat = &.{ vector, value, position } }, + .results = .{ .flat = &.{vector.type_()} }, + .location = location, + }); +} + +pub fn shufflevector(ctx: *mlir.Context, v1: *const mlir.Value, v2: *const mlir.Value, result_type: *const mlir.Type, mask: []const i32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.shufflevector", .{ + .operands = .{ .flat = &.{ v1, v2 } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "mask", .denseArray(ctx, .i32, mask))}, + .location = location, + }); +} + +pub fn mlir_undef(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.undef", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn mlir_poison(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.poison", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn freeze(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.freeze", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .location = location, + }); +} + +pub const BrOpts = struct { + loop_annotation: ?*const LoopAnnotationAttr = null, +}; + +/// `llvm.br`. Not verified: a terminator only verifies inside its block. +pub fn br(ctx: *mlir.Context, dest: *mlir.Block, dest_operands: []const *const mlir.Value, opts: BrOpts, location: *const mlir.Location) *mlir.Operation { + var state: mlir.OperationState = .init("llvm.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + if (opts.loop_annotation) |annotation| state.addAttributes(&.{.named(ctx, "loop_annotation", annotation.attribute())}); + return mlir.Operation.init(&state) catch @panic("llvm.br: failed to create the operation"); +} + +/// `llvm.unreachable`. Not verified: a terminator only verifies inside its +/// block. +pub fn unreachable_(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.unreachable", .{ .location = location, .verify = false }); +} + +/// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an +/// optional result of type `result_type`. +pub fn call_intrinsic( + ctx: *mlir.Context, + name: []const u8, + operands: []const *const mlir.Value, + opts: struct { + result_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.call_intrinsic", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, + .attributes = &.{ + .named(ctx, "intrin", .string(ctx, name)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ @intCast(operands.len), 0 })), + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + }, + .location = location, + }); +} + +pub const InlineAsmOpts = struct { + result_type: ?*const mlir.Type = null, + + /// LLVM constraint string, e.g. `"=r,r"` (outputs first). + constraints: []const u8 = "", + /// The assembly has effects beyond its outputs (it is never removed or merged). + has_side_effects: bool = false, +}; + +/// llvm.inline_asm — the PTX (or other target) assembly `asm_string`, with `$i` +/// placeholders bound to the result then `operands`. +pub fn inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + operands: []const *const mlir.Value, + opts: InlineAsmOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "asm_string", .string(ctx, asm_string))); + attrs.appendAssumeCapacity(.named(ctx, "constraints", .string(ctx, opts.constraints))); + if (opts.has_side_effects) { + attrs.appendAssumeCapacity(.named(ctx, "has_side_effects", .unit(ctx))); + } + return mlir.Operation.make(ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig new file mode 100644 index 00000000..6ba84b4d --- /dev/null +++ b/mlir/dialects/llvm/test.zig @@ -0,0 +1,210 @@ +const std = @import("std"); + +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "llvm.alloca", + "llvm.atomicrmw", + "llvm.bitcast", + "llvm.br", + "llvm.call_intrinsic", + "llvm.extractelement", + "llvm.extractvalue", + "llvm.fmul", + "llvm.freeze", + "llvm.getelementptr", + "llvm.inline_asm", + "llvm.insertelement", + "llvm.intr.assume", + "llvm.intr.lifetime.end", + "llvm.intr.lifetime.start", + "llvm.inttoptr", + "llvm.ptrtoint", + "llvm.load", + "llvm.mlir.poison", + "llvm.mlir.undef", + "llvm.shufflevector", + "llvm.store", + "llvm.unreachable", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(llvm); + inline for (comptime std.meta.declarations(llvm)) |decl| { + const T = @field(llvm, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "literal struct types build from their fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const struct_type = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<(i32, f32)>"))); +} + +test "identified struct types build from their name and fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const fields: []const *const mlir.Type = &.{ .int(ctx, .i32), .float(ctx, .f32) }; + const struct_type = llvm.identifiedStructType(ctx, "struct.args", fields).?; + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<\"struct.args\", (i32, f32)>"))); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields).?.eql(struct_type)); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields[0..1]) == null); +} + +test "getelementptr and alloca build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const ptr = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global); + const block = mlir.Block.init(&.{ ptr, .int(ctx, .i64) }, &.{ loc, loc }); + defer block.deinit(); + const array = llvm.arrayType(.float(ctx, .f32), 8); + const gep = llvm.getelementptr(ctx, block.argument(0), &.{ .{ .constant = 0 }, .{ .value = block.argument(1) } }, array, loc).appendTo(block); + try std.testing.expect(gep.verify()); + try std.testing.expect(gep.result(0).type_().eql(ptr)); + const alloca = llvm.alloca(ctx, block.argument(1), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), array, .{ .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(alloca.verify()); +} + +test "extractvalue, atomicrmw and intr.assume build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const pair = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + const block = mlir.Block.init(&.{ pair, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global), .int(ctx, .i1) }, &.{ loc, loc, loc }); + defer block.deinit(); + const first = llvm.extractvalue(ctx, block.argument(0), .int(ctx, .i32), &.{0}, loc).appendTo(block); + try std.testing.expect(first.verify()); + const previous = llvm.atomicrmw(ctx, block.argument(1), first.result(0), .add, .monotonic, .{ .syncscope = "agent", .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(previous.verify()); + try std.testing.expect(previous.result(0).type_().eql(.int(ctx, .i32))); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{previous}); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), "llvm.atomicrmw add ") != null); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), " monotonic ") != null); + try std.testing.expect(llvm.intr_assume(ctx, block.argument(2), loc).appendTo(block).verify()); +} + +test "vector, freeze and lifetime operations build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init(&.{ i64x2, .int(ctx, .i64), .int(ctx, .i32), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic) }, &.{ loc, loc, loc, loc }); + defer block.deinit(); + const v = block.argument(0); + const inserted = llvm.insertelement(ctx, v, block.argument(1), block.argument(2), loc).appendTo(block); + try std.testing.expect(inserted.verify()); + try std.testing.expect(inserted.result(0).type_().eql(i64x2)); + try std.testing.expect(llvm.extractelement(ctx, v, block.argument(2), .int(ctx, .i64), loc).appendTo(block).verify()); + const poison = llvm.mlir_poison(ctx, i64x2, loc).appendTo(block); + try std.testing.expect(poison.verify()); + try std.testing.expect(llvm.mlir_undef(ctx, i64x2, loc).appendTo(block).verify()); + try std.testing.expect(llvm.shufflevector(ctx, v, poison.result(0), i64x2, &.{ 1, 0 }, loc).appendTo(block).verify()); + try std.testing.expect(llvm.freeze(ctx, v, loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_start(ctx, block.argument(3), loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_end(ctx, block.argument(3), loc).appendTo(block).verify()); +} + +test "br carries its loop annotation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const dest = mlir.Block.init(&.{}, &.{}); + defer dest.deinit(); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .mustProgress = true }); + const op = llvm.br(ctx, dest, &.{}, .{ .loop_annotation = annotation }, loc); + defer op.deinit(); + try std.testing.expect(op.attributeByName("loop_annotation").?.eql(annotation.attribute())); + const end = llvm.unreachable_(ctx, loc); + defer end.deinit(); +} + +test "argument and function attributes are named by the LLVM dialect" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ + llvm.noundef(ctx), llvm.zeroext(ctx), llvm.signext(ctx), + llvm.alignment(ctx, 16), llvm.dereferenceable(ctx, 64), llvm.elementtype(ctx, .float(ctx, .f32)), + llvm.linkage(ctx, .internal), llvm.inlineHint(ctx), + })}); + try std.testing.expectEqualStrings("{llvm.align = 16 : i64, llvm.dereferenceable = 64 : i64, llvm.elementtype = f32, llvm.inline_hint, llvm.linkage = #llvm.linkage, llvm.noundef, llvm.signext, llvm.zeroext}", w.buffered()); +} + +test "emit_c_interface is a unit attribute of the LLVM dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{mlir.Attribute.dict(ctx, &.{llvm.emitCInterface(ctx)})}); + try std.testing.expectEqualStrings("{llvm.emit_c_interface}", writer.written()); +} + +test "loop annotations are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + const unroll = llvm.LoopUnrollAttr.get(ctx, .{ .full = true }); + try std.testing.expect(unroll.attribute().isA(llvm.LoopUnrollAttr) != null); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .unroll = unroll, .mustProgress = true }); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{annotation}); + try std.testing.expectEqualStrings("#llvm.loop_annotation, mustProgress = true>", writer.written()); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(annotation.attribute())); +} + +test "linkage attributes are built through the C API" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{llvm.linkageAttr(ctx, .weak_odr)}); + try std.testing.expectEqualStrings("#llvm.linkage", writer.written()); +} diff --git a/mlir/dialects/math/BUILD.bazel b/mlir/dialects/math/BUILD.bazel new file mode 100644 index 00000000..3845c969 --- /dev/null +++ b/mlir/dialects/math/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "math", + import_name = "mlir/dialects/math", + main = "math.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@llvm-project//mlir:CAPIMath", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":math", + "//mlir", + ], +) diff --git a/mlir/dialects/math/math.zig b/mlir/dialects/math/math.zig new file mode 100644 index 00000000..4ca6ee6d --- /dev/null +++ b/mlir/dialects/math/math.zig @@ -0,0 +1,210 @@ +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("math"); +} + +// ============================================================================= +// Float unary — SameOperandsAndResultType. Result type inferred from operand. +// ============================================================================= + +fn floatUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Exponentials / logs +pub const exp = floatUnaryOp("math.exp").call; +pub const exp2 = floatUnaryOp("math.exp2").call; +pub const expm1 = floatUnaryOp("math.expm1").call; +pub const log = floatUnaryOp("math.log").call; +pub const log2 = floatUnaryOp("math.log2").call; +pub const log10 = floatUnaryOp("math.log10").call; +pub const log1p = floatUnaryOp("math.log1p").call; + +// Trig / hyperbolic +pub const sin = floatUnaryOp("math.sin").call; +pub const cos = floatUnaryOp("math.cos").call; +pub const tan = floatUnaryOp("math.tan").call; +pub const asin = floatUnaryOp("math.asin").call; +pub const acos = floatUnaryOp("math.acos").call; +pub const atan = floatUnaryOp("math.atan").call; +pub const sinh = floatUnaryOp("math.sinh").call; +pub const cosh = floatUnaryOp("math.cosh").call; +pub const tanh = floatUnaryOp("math.tanh").call; +pub const asinh = floatUnaryOp("math.asinh").call; +pub const acosh = floatUnaryOp("math.acosh").call; +pub const atanh = floatUnaryOp("math.atanh").call; + +// Roots / errors / abs +pub const sqrt = floatUnaryOp("math.sqrt").call; +pub const rsqrt = floatUnaryOp("math.rsqrt").call; +pub const cbrt = floatUnaryOp("math.cbrt").call; +pub const erf = floatUnaryOp("math.erf").call; +pub const erfc = floatUnaryOp("math.erfc").call; +pub const absf = floatUnaryOp("math.absf").call; + +// Rounding +pub const floor = floatUnaryOp("math.floor").call; +pub const ceil = floatUnaryOp("math.ceil").call; +pub const round = floatUnaryOp("math.round").call; +pub const roundeven = floatUnaryOp("math.roundeven").call; +pub const trunc = floatUnaryOp("math.trunc").call; + +// ============================================================================= +// Float binary — SameOperandsAndResultType. +// ============================================================================= + +fn floatBinaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const powf = floatBinaryOp("math.powf").call; +pub const atan2 = floatBinaryOp("math.atan2").call; +pub const copysign = floatBinaryOp("math.copysign").call; + +// ============================================================================= +// Float ternary — SameOperandsAndResultType. +// ============================================================================= + +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fma", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .result_type_inference = true, + .location = location, + }); +} + +/// math.clampf value, min, max — clamp to [min, max], same type throughout. +/// Distinct from `tt.clampf` (which has an explicit `propagateNan` attr); the +/// math version uses fastmath for NaN behaviour. +pub fn clampf( + ctx: *mlir.Context, + value: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.clampf", .{ + .operands = .{ .flat = &.{ value, min, max } }, + .result_type_inference = true, + .location = location, + }); +} + +// ============================================================================= +// Sincos — operand → (sin, cos), both same type as operand. +// ============================================================================= + +pub fn sincos( + ctx: *mlir.Context, + operand: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.sincos", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{ operand.type_(), operand.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// FPowI — float base, signless integer exponent → float (same type as base). +// ============================================================================= + +pub fn fpowi( + ctx: *mlir.Context, + base: *const mlir.Value, + power: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fpowi", .{ + .operands = .{ .flat = &.{ base, power } }, + .results = .{ .flat = &.{base.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Integer unary +// ============================================================================= + +fn intUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const absi = intUnaryOp("math.absi").call; +pub const ctlz = intUnaryOp("math.ctlz").call; +pub const cttz = intUnaryOp("math.cttz").call; +pub const ctpop = intUnaryOp("math.ctpop").call; + +/// math.ipowi(base, exp) — integer power. Result type matches base. +pub const ipowi = floatBinaryOp("math.ipowi").call; + +// ============================================================================= +// Float classification — operand is float-like, result is i1-like. +// ============================================================================= + +fn classifyOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const isfinite = classifyOp("math.isfinite").call; +pub const isinf = classifyOp("math.isinf").call; +pub const isnan = classifyOp("math.isnan").call; +pub const isnormal = classifyOp("math.isnormal").call; diff --git a/mlir/dialects/math/test.zig b/mlir/dialects/math/test.zig new file mode 100644 index 00000000..fe2b7bc4 --- /dev/null +++ b/mlir/dialects/math/test.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "math.absf", + "math.absi", + "math.acos", + "math.acosh", + "math.asin", + "math.asinh", + "math.atan", + "math.atan2", + "math.atanh", + "math.cbrt", + "math.ceil", + "math.clampf", + "math.copysign", + "math.cos", + "math.cosh", + "math.ctlz", + "math.ctpop", + "math.cttz", + "math.erf", + "math.erfc", + "math.exp", + "math.exp2", + "math.expm1", + "math.floor", + "math.fma", + "math.fpowi", + "math.ipowi", + "math.isfinite", + "math.isinf", + "math.isnan", + "math.isnormal", + "math.log", + "math.log10", + "math.log1p", + "math.log2", + "math.powf", + "math.round", + "math.roundeven", + "math.rsqrt", + "math.sin", + "math.sincos", + "math.sinh", + "math.sqrt", + "math.tan", + "math.tanh", + "math.trunc", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(math); + inline for (comptime std.meta.declarations(math)) |decl| { + const T = @field(math, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + math.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/memref/BUILD.bazel b/mlir/dialects/memref/BUILD.bazel new file mode 100644 index 00000000..be1a16bb --- /dev/null +++ b/mlir/dialects/memref/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "memref", + import_name = "mlir/dialects/memref", + main = "memref.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIMemRef", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":memref", + "//mlir", + ], +) diff --git a/mlir/dialects/memref/memref.zig b/mlir/dialects/memref/memref.zig new file mode 100644 index 00000000..6c45eb33 --- /dev/null +++ b/mlir/dialects/memref/memref.zig @@ -0,0 +1,670 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("memref"); +} + +/// Sentinel matching `mlir::ShapedType::kDynamic` (== INT64_MIN). +/// Use this in static-size/offset/stride arrays to mark a slot as dynamic. +pub const kDynamic: i64 = std.math.minInt(i64); + +// ============================================================================= +// Allocation — alloc / alloca / dealloc / realloc +// ============================================================================= + +pub const AllocOpts = struct { + alignment: ?i64 = null, +}; + +pub const ReallocOpts = struct { + alignment: ?i64 = null, + dynamic_size: ?*const mlir.Value = null, +}; + +/// memref.alloc — heap allocation. `dynamic_sizes` provides one Index value per +/// dynamic dim of `result_type` (left-to-right). `symbol_operands` binds to +/// the symbols of the layout map (rare; pass &.{} when none). +pub fn alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// memref.alloca — stack allocation; same operand/attribute layout as alloc. +/// Unverified: it only verifies inside an `AutomaticAllocationScope`. +pub fn alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloca", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +pub fn dealloc( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dealloc", .{ + .operands = .{ .flat = &.{memref} }, + .location = location, + }); +} + +/// memref.realloc — grow/shrink an existing allocation. `dynamic_size` is +/// optional; omit `opts.dynamic_size` to keep the size encoded in `result_type`. +pub fn realloc( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ReallocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (opts.dynamic_size) |s| operands_buf.appendAssumeCapacity(s); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.realloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Load / Store — plain (non-affine) memref load/store +// ============================================================================= + +pub const LoadOpts = struct { + /// Hint that the lowering may use a non-temporal load. + nontemporal: bool = false, +}; + +/// memref.load — load a single element from a memref. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.load", .{ + .operands = .{ .flat = buf.constSlice() }, + .result_type_inference = true, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StoreOpts = struct { + nontemporal: bool = false, +}; + +/// memref.store — store a single element into a memref. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Atomic RMW +// ============================================================================= + +/// Matches `mlir::arith::AtomicRMWKind` (I64EnumAttr) in upstream `Arith` td. +/// memref.atomic_rmw and affine.parallel both encode reductions through this. +pub const AtomicRMWKind = enum(i64) { + addf = 0, + addi = 1, + andi = 2, + assign = 3, + maximumf = 4, + maxnumf = 5, + maxs = 6, + maxu = 7, + minimumf = 8, + minnumf = 9, + mins = 10, + minu = 11, + mulf = 12, + muli = 13, + ori = 14, + xori = 15, +}; + +/// memref.atomic_rmw — atomic read-modify-write. +pub fn atomic_rmw( + ctx: *mlir.Context, + kind: AtomicRMWKind, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.atomic_rmw", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{ + .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), + }, + .location = location, + }); +} + +// ============================================================================= +// Casts / metadata-only ops +// ============================================================================= + +/// memref.cast — change layout/element-shape compatibility (e.g. unranked ↔ +/// ranked, dynamic ↔ static). Element type and total memory must match. +pub fn cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.memory_space_cast — change the memory-space attribute on a memref. +pub fn memory_space_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.memory_space_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.assume_alignment — pure, returns the same memref with an alignment +/// hint attached. +pub fn assume_alignment( + ctx: *mlir.Context, + src: *const mlir.Value, + alignment: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.assume_alignment", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "alignment", .int(ctx, .i32, alignment)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape introspection +// ============================================================================= + +/// memref.dim — returns the size of dim `index` of `memref` as `index`. +pub fn dim( + ctx: *mlir.Context, + memref: *const mlir.Value, + index: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dim", .{ + .operands = .{ .flat = &.{ memref, index } }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// memref.rank — returns rank of `memref` as `index`. +pub fn rank( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.rank", .{ + .operands = .{ .flat = &.{memref} }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// memref.extract_aligned_pointer_as_index — get the aligned base pointer of +/// a memref as an `index`. +pub fn extract_aligned_pointer_as_index( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +// ============================================================================= +// View / SubView / Reinterpret +// ============================================================================= + +/// memref.view — type-changing view of a flat 1-D `memref` as a higher- +/// rank memref. `byte_shift` is an `index` byte offset; `dynamic_sizes` +/// supplies one operand per dynamic dim of `result_type`. +pub fn view( + ctx: *mlir.Context, + source: *const mlir.Value, + byte_shift: *const mlir.Value, + dynamic_sizes: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ source, byte_shift }); + buf.appendSliceAssumeCapacity(dynamic_sizes); + + return mlir.Operation.make(ctx, "memref.view", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// Mixed static/dynamic offsets-sizes-strides — `static_*` slots holding +/// `kDynamic` consume one entry from the corresponding `dynamic_*` slice. +pub const SubViewArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.subview — strided view into a source memref. The result type must +/// be precomputed by the caller (the dialect provides `inferResultType`, +/// but we don't expose that yet — pass an explicit `result_type`). +pub fn subview( + ctx: *mlir.Context, + source: *const mlir.Value, + args: SubViewArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.subview", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +pub const ReinterpretCastArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.reinterpret_cast — changes offset/sizes/strides of a memref without +/// touching the underlying memory. +pub fn reinterpret_cast( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReinterpretCastArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.reinterpret_cast", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Reshape — expand_shape / collapse_shape / reshape +// ============================================================================= + +/// memref.reshape — reshape a memref using a separate `shape` memref of `index` +/// elements. `result_type` carries the new shape. +pub fn reshape( + ctx: *mlir.Context, + source: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.reshape", .{ + .operands = .{ .flat = &.{ source, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `reassociation[i]` lists the source dimensions that map into result dim i +/// (for collapse_shape) or the result dimensions that come from source dim i +/// (for expand_shape). Each inner slice is stored as an `IntegerAttr` array. +pub const ReassociationArgs = struct { + /// Index groupings. + reassociation: []const []const i64, + /// For expand_shape only: per-result-dim sizes, with `kDynamic` for slots + /// resolved from `dynamic_output_shape`. Pass &.{} for collapse_shape. + static_output_shape: []const i64 = &.{}, + /// For expand_shape only: the dynamic dim values referenced by `kDynamic` + /// entries in `static_output_shape`. + dynamic_output_shape: []const *const mlir.Value = &.{}, +}; + +fn buildReassociationAttr(ctx: *mlir.Context, reassociation: []const []const i64) *const mlir.Attribute { + var outer: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; + var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; + for (reassociation) |group| { + inner_buf = .empty; + for (group) |idx| { + inner_buf.appendAssumeCapacity(.int(ctx, .i64, idx)); + } + outer.appendAssumeCapacity(.array(ctx, inner_buf.constSlice())); + } + return .array(ctx, outer.constSlice()); +} + +/// memref.expand_shape — split source dimensions into multiple result +/// dimensions. `static_output_shape` must have one entry per result dim; +/// dynamic entries are filled from `dynamic_output_shape` in order. +pub fn expand_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReassociationArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_output_shape); + + return mlir.Operation.make(ctx, "memref.expand_shape", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, args.reassociation)), + .named(ctx, "static_output_shape", .denseArray(ctx, .i64, args.static_output_shape)), + }, + .location = location, + }); +} + +/// memref.collapse_shape — merge consecutive source dimensions into single +/// result dimensions. Reassociation map is canonical. +pub fn collapse_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + reassociation: []const []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.collapse_shape", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, reassociation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Transpose — metadata-only permutation. Permutation is an AffineMapAttr. +// ============================================================================= + +/// memref.transpose — permute the dimensions of `src` according to +/// `permutation` (which must be a permutation of `(d0, ..., dN-1)`). Build +/// the map with `mlir.AffineMap.get` or parse via +/// `mlir.parseAffineMapAttribute`. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: *const mlir.Attribute, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", permutation), + }, + .location = location, + }); +} + +// ============================================================================= +// Copy / Prefetch +// ============================================================================= + +/// memref.copy — copy source memref into target memref (same element type, +/// same total element count). +pub fn copy( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.copy", .{ + .operands = .{ .flat = &.{ source, target } }, + .location = location, + }); +} + +pub const PrefetchOpts = struct { + /// True for a write prefetch, false for a read prefetch. + is_write: bool = false, + /// 0 = no locality, 3 = extreme locality (keep in cache). + locality_hint: i32 = 3, + /// True for data cache, false for instruction cache. + is_data_cache: bool = true, +}; + +/// memref.prefetch — non-binding hint to bring `memref[indices]` into cache. +pub fn prefetch( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: PrefetchOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.prefetch", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "isWrite", .boolean(ctx, opts.is_write)), + .named(ctx, "localityHint", .int(ctx, .i32, opts.locality_hint)), + .named(ctx, "isDataCache", .boolean(ctx, opts.is_data_cache)), + }, + .location = location, + }); +} + +// ============================================================================= +// Globals +// ============================================================================= + +pub const GlobalArgs = struct { + pub const Visibility = enum { public, private, nested }; + + name: []const u8, + type_: *const mlir.Type, + /// Optional dense initial value. `null` ⇒ uninitialised global. + initial_value: ?*const mlir.Attribute = null, + visibility: Visibility = .public, + constant: bool = false, + alignment: ?i64 = null, +}; + +/// memref.global — declare a module-scope memref global (matches the symbol +/// referenced by `memref.get_global`). +pub fn global(ctx: *mlir.Context, args: GlobalArgs, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "type", .typeAttr(args.type_)), + }); + if (args.initial_value) |iv| { + attrs.appendAssumeCapacity(.named(ctx, "initial_value", iv)); + } + if (args.constant) { + attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); + } + if (args.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.global", .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// memref.get_global — fetch the value of a `memref.global` by symbol name. +pub fn get_global( + ctx: *mlir.Context, + name: []const u8, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.get_global", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "name", .flatSymbolRef(ctx, name)), + }, + .location = location, + }); +} diff --git a/mlir/dialects/memref/test.zig b/mlir/dialects/memref/test.zig new file mode 100644 index 00000000..cddb74c5 --- /dev/null +++ b/mlir/dialects/memref/test.zig @@ -0,0 +1,77 @@ +const std = @import("std"); + +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "memref.alloc", + "memref.alloca", + "memref.assume_alignment", + "memref.atomic_rmw", + "memref.cast", + "memref.collapse_shape", + "memref.copy", + "memref.dealloc", + "memref.dim", + "memref.expand_shape", + "memref.extract_aligned_pointer_as_index", + "memref.get_global", + "memref.global", + "memref.load", + "memref.memory_space_cast", + "memref.prefetch", + "memref.rank", + "memref.realloc", + "memref.reinterpret_cast", + "memref.reshape", + "memref.store", + "memref.subview", + "memref.transpose", + "memref.view", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(memref); + inline for (comptime std.meta.declarations(memref)) |decl| { + const T = @field(memref, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "atomic_rmw kinds print as their IR spelling" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{ .float(ctx, .f32), try mlir.Type.parse(ctx, "memref<4xf32>"), .int(ctx, .i32), try mlir.Type.parse(ctx, "memref<4xi32>"), .index(ctx) }, &.{ loc, loc, loc, loc, loc }); + defer block.deinit(); + inline for (comptime std.meta.tags(memref.AtomicRMWKind)) |kind| { + const float = switch (kind) { + .addf, .assign, .maximumf, .maxnumf, .minimumf, .minnumf, .mulf => true, + else => false, + }; + const value = block.argument(if (float) 0 else 2); + const ref = block.argument(if (float) 1 else 3); + const op = memref.atomic_rmw(ctx, kind, value, ref, &.{block.argument(4)}, loc); + defer op.deinit(); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{op}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "memref.atomic_rmw " ++ @tagName(kind) ++ " ") != null); + } +} diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..88e67265 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +# The types and attributes Mosaic's C API does not expose. +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:CAPIIR", + "@xla//xla/mosaic:tpu_dialect", + ], +) + +zig_library( + name = "mosaic_tpu", + import_name = "mlir/dialects/mosaic_tpu", + main = "mosaic_tpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@xla//xla/mosaic:tpu_dialect_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":mosaic_tpu", + "//mlir", + ], +) diff --git a/mlir/dialects/mosaic_tpu/capi.cc b/mlir/dialects/mosaic_tpu/capi.cc new file mode 100644 index 00000000..22662082 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/capi.cc @@ -0,0 +1,273 @@ +//===- capi.cc - C API for the Mosaic tpu dialect -------------------------===// + +#include "capi.h" + +#include "xla/mosaic/dialect/tpu/tpu_dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/ArrayRef.h" + +#include +#include + +static mlir::MLIRContext *loadTPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +template +static MlirAttribute enumAttrGet(MlirContext context, + std::optional value) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx || !value) + return {nullptr}; + return wrap(AttrT::get(ctx, *value)); +} + +template static uint32_t enumAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +static llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { + return llvm::ArrayRef(data, static_cast(size)); +} + +bool mlirTypeIsATpuSemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirTpuSemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::SemaphoreType::get(ctx)); +} + +bool mlirTypeIsATpuDMASemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::DMASemaphoreType::get(ctx)); +} + +bool mlirAttributeIsATpuReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeReductionKind(value)); +} + +uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeContractPrecision(value)); +} + +uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeRoundingMode(value)); +} + +uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuCoreType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeCoreType(value)); +} + +uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeDimensionSemantics(value)); +} + +uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizePipelineMode(value)); +} + +uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeRevisitMode(value)); +} + +uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, + int64_t coreType) { + mlir::MLIRContext *ctx = loadTPU(context); + std::optional memorySpace = + mlir::tpu::symbolizeMemorySpace(value); + if (!ctx || !memorySpace) + return {nullptr}; + std::optional core; + if (coreType >= 0) { + if (coreType > UINT32_MAX) + return {nullptr}; + core = mlir::tpu::symbolizeCoreType(static_cast(coreType)); + if (!core) + return {nullptr}; + } + return wrap(mlir::tpu::MemorySpaceAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *memorySpace, core)); +} + +uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr) { + std::optional core = + llvm::cast(unwrap(attr)).getCoreType(); + return core ? static_cast(*core) : -1; +} + +bool mlirAttributeIsATpuElementWindow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, + intptr_t numPadLow, + const int64_t *padLow, + intptr_t numPadHigh, + const int64_t *padHigh) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::ElementWindowAttr::get( + ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); +} + +intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getPadLow() + .size(); +} + +int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)) + .getPadLow()[pos]; +} + +intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getPadHigh() + .size(); +} + +int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)) + .getPadHigh()[pos]; +} + +bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::DotDimensionNumbersAttr::get( + ctx, arrayRef(numLhsContractingDims, lhsContractingDims), + arrayRef(numRhsContractingDims, rhsContractingDims), + arrayRef(numLhsNonContractingDims, lhsNonContractingDims), + arrayRef(numRhsNonContractingDims, rhsNonContractingDims), + arrayRef(numOutputDimOrder, outputDimOrder), + arrayRef(numLhsBatchDims, lhsBatchDims), + arrayRef(numRhsBatchDims, rhsBatchDims))); +} + +#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ + intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name() \ + .size(); \ + } \ + int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, \ + intptr_t pos) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name()[pos]; \ + } + +ZML_TPU_DOT_DIMS_GETTERS(LhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(LhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(OutputDimOrder) +ZML_TPU_DOT_DIMS_GETTERS(LhsBatchDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsBatchDims) + +#undef ZML_TPU_DOT_DIMS_GETTERS diff --git a/mlir/dialects/mosaic_tpu/capi.h b/mlir/dialects/mosaic_tpu/capi.h new file mode 100644 index 00000000..2e1cc0bb --- /dev/null +++ b/mlir/dialects/mosaic_tpu/capi.h @@ -0,0 +1,145 @@ +//===- capi.h - C API for the Mosaic tpu dialect ------------------*- C -*-===// +// +// C entry points for the `tpu` dialect that Mosaic's C API +// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes +// kernels build. + +#ifndef MLIR_DIALECTS_MOSAIC_TPU_CAPI_H +#define MLIR_DIALECTS_MOSAIC_TPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// !tpu.semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); + +// !tpu.dma_semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); + +// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); +// `Get` returns a null attribute for a value outside the enum. + +// #tpu.reduction_kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuReductionKindAttrGetValue(MlirAttribute attr); + +// #tpu.contract_precision<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuContractPrecision(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); + +// #tpu.rounding_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); + +// #tpu.core_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); + +// #tpu.dimension_semantics<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); + +// #tpu.pipeline_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); + +// #tpu.revisit_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); + +// #tpu.memory_space: a negative `coreType` leaves the +// optional core type out, and `GetCoreType` returns -1 when it is absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, + uint32_t value, + int64_t coreType); +MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); + +// #tpu.element_window<[pad_low], [pad_high]> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet( + MlirContext context, intptr_t numPadLow, const int64_t *padLow, + intptr_t numPadHigh, const int64_t *padHigh); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); + +// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..8425e7b4 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,1556 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// Mosaic IR version stored as `stable_mosaic.version`, matching +/// `xla/mosaic/dialect/tpu/transforms/serde.h:kVersion`. This is distinct from +/// the custom-call JSON `serialization_format`, which remains 1. +pub const SERDE_VERSION: i32 = 18; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tpu"); +} + +/// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by +/// name. Idempotent — safe to call repeatedly. Required before running the +/// `mosaic-serde{serialize=true}` pipeline that produces the bytecode embedded +/// in `tpu_custom_call`'s backend_config. +pub fn registerMosaicSerdePass() void { + c.mlirTpuRegisterMosaicSerdePass(); +} + +// ============================================================================= +// Types — !tpu.semaphore, !tpu.dma_semaphore, !tpu.float8_exmy<...> +// ============================================================================= + +/// `!tpu.semaphore`. +pub const SemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuSemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "semaphore"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuSemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!tpu.semaphore` element type. +pub fn semaphoreType(ctx: *mlir.Context) *const mlir.Type { + const ty = SemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); +} + +/// `!tpu.dma_semaphore`. +pub const DMASemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuDMASemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "dma_semaphore"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuDMASemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// Build `!tpu.dma_semaphore`. +pub fn dmaSemaphoreType(ctx: *mlir.Context) *const mlir.Type { + const ty = DMASemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); +} + +/// `!tpu.float8_exmy` — Mosaic Float8 type. +pub const Float8EXMYType = opaque { + const M = mlir.Methods(Float8EXMYType, c.MlirType); + + pub const isAFn = c.mlirTpuIsAFloat8EXMYType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, underlying: *const mlir.Type) *const Float8EXMYType { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGet(ctx.ptr(), underlying.ptr()).ptr); + } + + pub fn underlyingType(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGetUnderlyingType(self.ptr()).ptr); + } + + pub fn type_(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const mlir.Type { + return @ptrCast(Float8EXMYType.get(ctx, underlying)); +} + +// ============================================================================= +// Attributes +// ============================================================================= + +/// Reduction kind for `tpu.all_reduce`, `tpu.reduce_index`, `tpu.scan`. +pub const ReductionKind = enum(u32) { + sum = 0, + max = 1, + min = 2, + arg_max = 3, + arg_min = 4, + find_first_set = 5, + maxf = 6, + minf = 7, + maxsi = 8, + minsi = 9, + maxui = 10, + minui = 11, + + pub fn attribute(self: ReductionKind, ctx: *mlir.Context) *const mlir.Attribute { + const attr = ReductionKindAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } + + fn forInput(self: ReductionKind, input: *const mlir.Value, unsigned_integer: bool) ReductionKind { + if (self != .max and self != .min) return self; + const element_type = input.type_().isA(mlir.ShapedType).?.elementType(); + if (element_type.isFloat()) return if (self == .max) .maxf else .minf; + // Match Mosaic's serde upgrade: scans used unsigned integer extrema, + // while all-reduce used signed integer extrema. + return if (unsigned_integer) + (if (self == .max) .maxui else .minui) + else + (if (self == .max) .maxsi else .minsi); + } +}; + +/// `#tpu.reduction_kind<...>`. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirTpuReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// Matmul precision setting of `tpu.matmul`. +pub const ContractPrecision = enum(u32) { + bf16 = 0, + fp32 = 1, + bf16x3 = 2, + + pub fn attribute(self: ContractPrecision, ctx: *mlir.Context) *const mlir.Attribute { + const attr = ContractPrecisionAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.contract_precision<...>`. +pub const ContractPrecisionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuContractPrecision; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "contract_precision"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ContractPrecision, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ContractPrecision { + return @fromBackingInt(@intCast(c.mlirTpuContractPrecisionAttrGetValue(self.ptr()))); + } +}; + +/// Rounding mode of `tpu.fptosi` / `tpu.fptoui` / etc. +pub const RoundingMode = enum(u32) { + towards_zero = 0, + to_nearest_even = 1, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = RoundingModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.rounding_mode<...>`. +pub const RoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "rounding_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) RoundingMode { + return @fromBackingInt(@intCast(c.mlirTpuRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// Target core of a kernel, remote DMAs and `tpu.sem_signal`. +pub const CoreType = enum(u32) { + tc = 0, + sc_scalar_subcore = 1, + sc_vector_subcore = 2, + + pub fn attribute(self: CoreType, ctx: *mlir.Context) *const mlir.Attribute { + const attr = CoreTypeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.core_type<...>`. +pub const CoreTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuCoreType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "core_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CoreType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CoreType { + return @fromBackingInt(@intCast(c.mlirTpuCoreTypeAttrGetValue(self.ptr()))); + } +}; + +/// Grid dimension semantics annotation. +pub const DimensionSemantics = enum(u32) { + parallel = 0, + arbitrary = 1, + core_parallel = 2, + subcore_parallel = 3, + + pub fn attribute(self: DimensionSemantics, ctx: *mlir.Context) *const mlir.Attribute { + const attr = DimensionSemanticsAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.dimension_semantics<...>`. +pub const DimensionSemanticsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDimensionSemantics; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dimension_semantics"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DimensionSemantics, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) DimensionSemantics { + return @fromBackingInt(@intCast(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr()))); + } +}; + +/// Buffering mode of a `window_params` entry. +pub const PipelineMode = enum(u32) { + synchronous = 1, + double_buffered = 2, + + pub fn attribute(self: PipelineMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = PipelineModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.pipeline_mode<...>`. +pub const PipelineModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuPipelineMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "pipeline_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PipelineMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PipelineMode { + return @fromBackingInt(@intCast(c.mlirTpuPipelineModeAttrGetValue(self.ptr()))); + } +}; + +/// Block revisit mode of a `window_params` entry. +pub const RevisitMode = enum(u32) { + immediate = 0, + any = 1, + + pub fn attribute(self: RevisitMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = RevisitModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.revisit_mode<...>`. +pub const RevisitModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRevisitMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "revisit_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RevisitMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) RevisitMode { + return @fromBackingInt(@intCast(c.mlirTpuRevisitModeAttrGetValue(self.ptr()))); + } +}; + +/// Memory space of a memref, `#tpu.memory_space<...>`. +pub const MemorySpace = enum(u32) { + any = 0xffff_ffff, + vmem = 0, + smem = 1, + hbm = 2, + cmem = 3, + semaphore_mem = 4, + vmem_shared = 5, + host = 6, + + pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { + const attr = MemorySpaceAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.memory_space` — passed as the `memory_space` of a memref's type. +pub const MemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemorySpace, + coreType: ?CoreType = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const core_type: i64 = if (args.coreType) |ct| @backingInt(ct) else -1; + const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value), core_type); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemorySpace { + return @fromBackingInt(@intCast(c.mlirTpuMemorySpaceAttrGetValue(self.ptr()))); + } + pub fn coreType(self: *const Self) ?CoreType { + const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); + return if (core_type < 0) null else @fromBackingInt(@intCast(core_type)); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub const ElementWindowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuElementWindow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "element_window"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + padLow: []const i64, + padHigh: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuElementWindowAttrGet( + ctx.ptr(), + @intCast(args.padLow.len), + args.padLow.ptr, + @intCast(args.padHigh.len), + args.padHigh.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numPadLow(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadLow(self.ptr())); + } + pub fn padLow(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadLow(self.ptr(), @intCast(pos)); + } + pub fn numPadHigh(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadHigh(self.ptr())); + } + pub fn padHigh(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadHigh(self.ptr(), @intCast(pos)); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub fn elementWindowAttr( + ctx: *mlir.Context, + pad_low: []const i64, + pad_high: []const i64, +) *const mlir.Attribute { + const attr = ElementWindowAttr.get(ctx, .{ .padLow = pad_low, .padHigh = pad_high }) catch unreachable; + return attr.attribute(); +} + +/// `#tpu.dot_dimension_numbers<...>` — the dimension mapping of `tpu.matmul`. +pub const DotDimensionNumbersAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dot_dimension_numbers"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lhsContractingDims: []const i64, + rhsContractingDims: []const i64, + lhsNonContractingDims: []const i64, + /// Empty when rhs is a 1-D vector. + rhsNonContractingDims: []const i64 = &.{}, + /// Flattened (operand, dim) pairs: operand 0 is lhs, 1 is rhs. + outputDimOrder: []const i64, + lhsBatchDims: []const i64 = &.{}, + rhsBatchDims: []const i64 = &.{}, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDotDimensionNumbersAttrGet( + ctx.ptr(), + @intCast(args.lhsContractingDims.len), + args.lhsContractingDims.ptr, + @intCast(args.rhsContractingDims.len), + args.rhsContractingDims.ptr, + @intCast(args.lhsNonContractingDims.len), + args.lhsNonContractingDims.ptr, + @intCast(args.rhsNonContractingDims.len), + args.rhsNonContractingDims.ptr, + @intCast(args.outputDimOrder.len), + args.outputDimOrder.ptr, + @intCast(args.lhsBatchDims.len), + args.lhsBatchDims.ptr, + @intCast(args.rhsBatchDims.len), + args.rhsBatchDims.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numLhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(self.ptr())); + } + pub fn lhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(self.ptr())); + } + pub fn rhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numLhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(self.ptr())); + } + pub fn lhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(self.ptr())); + } + pub fn rhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numOutputDimOrder(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(self.ptr())); + } + pub fn outputDimOrder(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(self.ptr(), @intCast(pos)); + } + pub fn numLhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(self.ptr())); + } + pub fn lhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(self.ptr())); + } + pub fn rhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(self.ptr(), @intCast(pos)); + } +}; + +// ============================================================================= +// Discardable attributes — tpu.core_type, tpu.dimension_semantics +// ============================================================================= + +/// `tpu.core_type`, on a kernel's `func.func`: the core it runs on. +pub fn coreTypeNamedAttr(ctx: *mlir.Context, value: CoreType) mlir.NamedAttribute { + return .named(ctx, "tpu.core_type", value.attribute(ctx)); +} + +/// `tpu.dimension_semantics`, on a kernel's iteration index argument. +pub fn dimensionSemanticsNamedAttr(ctx: *mlir.Context, value: DimensionSemantics) mlir.NamedAttribute { + return .named(ctx, "tpu.dimension_semantics", value.attribute(ctx)); +} + +// ============================================================================= +// Reductions / scan / sort +// ============================================================================= + +pub fn all_reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + dim: i64, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.all_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i64, dim)), + .named(ctx, "kind", kind.forInput(input, false).attribute(ctx)), + }, + .location = location, + }); +} + +pub fn reduce_index( + ctx: *mlir.Context, + input: *const mlir.Value, + axis: i32, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reduce_index", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +pub const ScanOpts = struct { + mask: ?*const mlir.Value = null, +}; + +/// Scan the last dimension; an optional rank-one mask spans that dimension. +pub fn scan( + ctx: *mlir.Context, + input: *const mlir.Value, + kind: ReductionKind, + result_type: *const mlir.Type, + opts: ScanOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(input); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.scan", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.forInput(input, true).attribute(ctx)), + .named(ctx, "dimension", .int(ctx, .i64, @as(i64, @intCast(input.type_().isA(mlir.ShapedType).?.rank())) - 1)), + }, + .location = location, + }); +} + +pub const SortOpts = struct { + mask: ?*const mlir.Value = null, + + descending: bool = false, +}; + +pub fn sort( + ctx: *mlir.Context, + keys: *const mlir.Value, + values: *const mlir.Value, + output_mask_type: *const mlir.Type, + sorted_keys_type: *const mlir.Type, + sorted_values_type: *const mlir.Type, + opts: SortOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ keys, values }); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.sort", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{ output_mask_type, sorted_keys_type, sorted_values_type } }, + .attributes = &.{ + .named(ctx, "descending", .boolean(ctx, opts.descending)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory — load / store / vector_load / vector_store / strided_* +// ============================================================================= + +pub const LoadOpts = struct { + sublane_mask: []const bool, + sublane_stride: i32 = 1, +}; + +/// tpu.load — sublane-aware load into a vreg. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: LoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), + }, + .location = location, + }); +} + +pub const StoreOpts = struct { + mask: ?*const mlir.Value = null, + + sublane_mask: []const bool, + sublane_stride: i32 = 1, + add: bool = false, +}; + +/// tpu.store — sublane-aware store from a vreg. Optional elementwise mask. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), + .named(ctx, "add", .boolean(ctx, opts.add)), + }, + .location = location, + }); +} + +pub const VectorLoadOpts = struct { + mask: ?*const mlir.Value = null, + + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, +}; + +/// tpu.vector_load — multi-dim strided load with optional elementwise mask. +pub fn vector_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: VectorLoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), + }, + .location = location, + }); +} + +pub const VectorStoreOpts = struct { + mask: ?*const mlir.Value = null, + + strides: []const i32 = &.{}, + add: bool = false, +}; + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: VectorStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), + .named(ctx, "add", .boolean(ctx, opts.add)), + }, + .location = location, + }); +} + +pub fn strided_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), + }, + .location = location, + }); +} + +pub fn strided_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute — matmul, iota, reciprocal, casts +// ============================================================================= + +pub const MatmulOpts = struct { + /// Deprecated when `dimension_numbers` is provided. + transpose_lhs: bool = false, + transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, + /// Optional precision; pass null to omit. + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `DotDimensionNumbersAttr` or + /// `dotDimensionNumbersAttr`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub fn matmul( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + result_type: *const mlir.Type, + opts: MatmulOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "transpose_lhs", .boolean(ctx, opts.transpose_lhs)), + .named(ctx, "transpose_rhs", .boolean(ctx, opts.transpose_rhs)), + .named(ctx, "transpose_lhs_hint", .boolean(ctx, opts.transpose_lhs_hint)), + }); + if (opts.precision) |p| { + attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); + } + if (opts.dimension_numbers) |dn| { + attrs.appendAssumeCapacity(.named(ctx, "dimension_numbers", dn)); + } + + return mlir.Operation.make(ctx, "tpu.matmul", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const IotaOpts = struct { + dimensions: ?[]const i32 = null, +}; + +pub fn iota( + ctx: *mlir.Context, + result_type: *const mlir.Type, + opts: IotaOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dimensions) |d| { + attrs.appendAssumeCapacity(.named(ctx, "dimensions", .denseArray(ctx, .i32, d))); + } + return mlir.Operation.make(ctx, "tpu.iota", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub fn reciprocal( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: ReciprocalOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reciprocal", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "approx", .boolean(ctx, opts.approx)), + .named(ctx, "full_range", .boolean(ctx, opts.full_range)), + }, + .location = location, + }); +} + +/// Generic 1-operand → 1-result builder for the various TPU casts. +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const fptosi = castOp("tpu.fptosi").call; +pub const fptoui = castOp("tpu.fptoui").call; +pub const sitofp = castOp("tpu.sitofp").call; +pub const uitofp = castOp("tpu.uitofp").call; +pub const extf = castOp("tpu.extf").call; +pub const truncf = castOp("tpu.truncf").call; +pub const bitcast = castOp("tpu.bitcast").call; +pub const bitcast_vreg = castOp("tpu.bitcast_vreg").call; +pub const mask_cast = castOp("tpu.mask_cast").call; + +// ============================================================================= +// Shape — reshape / repeat / concatenate / transpose / broadcast_in_sublanes +// ============================================================================= + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn concatenate( + ctx: *mlir.Context, + sources: []const *const mlir.Value, + dimension: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.concatenate", .{ + .operands = .{ .flat = sources }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i32, dimension)), + }, + .location = location, + }); +} + +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), + }, + .location = location, + }); +} + +pub fn broadcast_in_sublanes( + ctx: *mlir.Context, + src: *const mlir.Value, + lane: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.broadcast_in_sublanes", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "lane", .int(ctx, .i32, lane)), + }, + .location = location, + }); +} + +pub const DynamicRotateOpts = struct { + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub fn dynamic_rotate( + ctx: *mlir.Context, + value: *const mlir.Value, + amount: *const mlir.Value, + opts: DynamicRotateOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "dimension", .int(ctx, .si32, opts.dimension)), + }); + if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .si32, s))); + if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .si32, sd))); + + return mlir.Operation.make(ctx, "tpu.dynamic_rotate", .{ + .operands = .{ .flat = &.{ value, amount } }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Memref-shaped ops — slice / squeeze / reshape / bitcast / reinterpret_cast +// ============================================================================= + +pub const MemRefSliceArgs = struct { + /// `base_idx` count must equal source memref rank. + base_idx: []const *const mlir.Value, + /// Optional dynamic sizes; left empty when target shape is fully static. + dynamic_sizes: []const *const mlir.Value = &.{}, +}; + +pub fn memref_slice( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + args: MemRefSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(mem_ref); + operands_buf.appendSliceAssumeCapacity(args.base_idx); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + + const seg_sizes = [3]i32{ 1, @intCast(args.base_idx.len), @intCast(args.dynamic_sizes.len) }; + return mlir.Operation.make(ctx, "tpu.memref_slice", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn memref_squeeze( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_squeeze", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_reshape( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_reshape", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_bitcast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_bitcast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reinterpret_cast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reinterpret_cast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 0, 0, 0 }))}, + .location = location, + }); +} + +pub fn assume_layout( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn erase_memref_layout( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.erase_memref_layout", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Mask building — create_mask / create_subelement_mask / mask_cast +// ============================================================================= + +/// `tpu.create_mask` — two equal-length operand groups (low, high) per dim. +pub fn create_mask( + ctx: *mlir.Context, + low_bounds: []const *const mlir.Value, + high_bounds: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(low_bounds.len == high_bounds.len); + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(low_bounds); + operands_buf.appendSliceAssumeCapacity(high_bounds); + + return mlir.Operation.make(ctx, "tpu.create_mask", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Semaphores / DMA / barriers +// ============================================================================= + +pub fn sem_alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_alloc", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_barrier( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_barrier", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_read( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_read", .{ + .operands = .{ .flat = &.{semaphore} }, + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .location = location, + }); +} + +pub fn sem_wait( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_wait", .{ + .operands = .{ .flat = &.{ semaphore, amount } }, + .location = location, + }); +} + +pub const SemSignalOpts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, +}; + +pub fn sem_signal( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + opts: SemSignalOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, amount }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, dev_len, core_len, subcore_len }; + return mlir.Operation.make(ctx, "tpu.sem_signal", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn barrier( + ctx: *mlir.Context, + barrier_id: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.barrier", .{ + .operands = .{ .flat = &.{barrier_id} }, + .location = location, + }); +} + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?*const mlir.Value = null, + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub fn enqueue_dma( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + target_semaphore: *const mlir.Value, + opts: EnqueueDmaOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 7) = .empty; + operands_buf.appendAssumeCapacity(source); + const src_sem_len: i32 = if (opts.source_semaphore) |s| blk: { + operands_buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + operands_buf.appendAssumeCapacity(target); + operands_buf.appendAssumeCapacity(target_semaphore); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? / subcore_id? + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [7]i32{ 1, src_sem_len, 1, 1, dev_len, core_len, subcore_len }; + return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "priority", .int(ctx, .i32, opts.priority)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +pub const WaitDma2Opts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + strict_ordering: bool = false, +}; + +pub fn wait_dma2( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: WaitDma2Opts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, src, dst }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, dev_len, core_len }; + return mlir.Operation.make(ctx, "tpu.wait_dma2", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +// ============================================================================= +// Region-bearing — region / trace / yield +// ============================================================================= + +/// tpu.region — single-block region with an implicit `tpu.yield` terminator. +pub fn region( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn trace( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn trace_start( + ctx: *mlir.Context, + level: i32, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_start", .{ + .attributes = &.{ + .named(ctx, "level", .int(ctx, .i32, level)), + .named(ctx, "message", .string(ctx, message)), + }, + .verify = false, + .location = location, + }); +} + +pub fn trace_stop(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_stop", .{ + .verify = false, + .location = location, + }); +} + +/// Build a `#tpu.dot_dimension_numbers<...>` attribute. Mirrors +/// `tpu_ops.td:TPU_DotDimensionNumbersAttr`. Pass empty slices for the +/// fields you don't use (most dot operations leave batch dims empty). +pub fn dotDimensionNumbersAttr( + ctx: *mlir.Context, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, +) *const mlir.Attribute { + const attr = DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = lhs_contracting, + .rhsContractingDims = rhs_contracting, + .lhsNonContractingDims = lhs_non_contracting, + .rhsNonContractingDims = rhs_non_contracting, + .outputDimOrder = output_dim_order, + .lhsBatchDims = lhs_batch, + .rhsBatchDims = rhs_batch, + }) catch unreachable; + return attr.attribute(); +} + +// ============================================================================= +// Misc — device_id / delay / assume_multiple +// ============================================================================= + +pub fn device_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.device_id", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .location = location, + }); +} + +pub fn delay( + ctx: *mlir.Context, + cycles: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.delay", .{ + .operands = .{ .flat = &.{cycles} }, + .location = location, + }); +} + +pub fn assume_multiple( + ctx: *mlir.Context, + src: *const mlir.Value, + multiple: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_multiple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "multiple", .int(ctx, .i32, multiple)), + }, + .location = location, + }); +} + +// ============================================================================= +// Internal helpers +// ============================================================================= + +/// `tpu.DenseBoolArrayAttr` stores one C `int` (i32) per bool, see `mlirDenseBoolArrayGet` +fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { + var ints: stdx.BoundedArray(i32, 64) = .empty; + for (values) |b| ints.appendAssumeCapacity(@intFromBool(b)); + return .denseArray(ctx, .bool, ints.constSlice()); +} diff --git a/mlir/dialects/mosaic_tpu/test.zig b/mlir/dialects/mosaic_tpu/test.zig new file mode 100644 index 00000000..20a20f42 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/test.zig @@ -0,0 +1,281 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); + +const operation_names: []const []const u8 = &.{ + "tpu.all_reduce", + "tpu.assume_layout", + "tpu.assume_multiple", + "tpu.barrier", + "tpu.bitcast", + "tpu.bitcast_vreg", + "tpu.broadcast_in_sublanes", + "tpu.concatenate", + "tpu.create_mask", + "tpu.delay", + "tpu.device_id", + "tpu.dynamic_rotate", + "tpu.enqueue_dma", + "tpu.erase_memref_layout", + "tpu.extf", + "tpu.fptosi", + "tpu.fptoui", + "tpu.iota", + "tpu.load", + "tpu.mask_cast", + "tpu.matmul", + "tpu.memref_bitcast", + "tpu.memref_reshape", + "tpu.memref_slice", + "tpu.memref_squeeze", + "tpu.reciprocal", + "tpu.reduce_index", + "tpu.region", + "tpu.reinterpret_cast", + "tpu.reshape", + "tpu.scan", + "tpu.sem_alloc", + "tpu.sem_barrier", + "tpu.sem_read", + "tpu.sem_signal", + "tpu.sem_wait", + "tpu.sitofp", + "tpu.sort", + "tpu.store", + "tpu.strided_load", + "tpu.strided_store", + "tpu.trace", + "tpu.trace_start", + "tpu.trace_stop", + "tpu.transpose", + "tpu.truncf", + "tpu.uitofp", + "tpu.vector_load", + "tpu.vector_store", + "tpu.wait_dma2", + "tpu.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { + const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; + try std.testing.expectEqual(expected.len, segments.numElements()); + for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { + const Enum = @FieldType(Attr.InitArgs, "value"); + inline for (@typeInfo(Enum).@"enum".field_names) |name| { + const value: Enum = @field(Enum, name); + const attr = try Attr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(Attr) != null); + try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); + try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); + } +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(mosaic_tpu); + inline for (comptime std.meta.declarations(mosaic_tpu)) |decl| { + const T = @field(mosaic_tpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "DMA and semaphore wrappers satisfy current operand groups" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const amount = args.addArgument(.int(ctx, .i32), loc); + + const dma = mosaic_tpu.enqueue_dma(ctx, source, target, dma_sem, .{}, loc); + defer dma.deinit(); + try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); + try std.testing.expect(dma.verify()); + const signal = mosaic_tpu.sem_signal(ctx, sem, amount, .{}, loc); + defer signal.deinit(); + try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); + try std.testing.expect(signal.verify()); + + const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const core = args.addArgument(.int(ctx, .i32), loc); + const subcore = args.addArgument(.int(ctx, .i32), loc); + const remote_signal = mosaic_tpu.sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); + defer remote_signal.deinit(); + try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_signal.operand(3).eql(subcore)); + const remote_dma = mosaic_tpu.enqueue_dma(ctx, source, target, sc_dma_sem, .{ + .source_semaphore = dma_sem, + .core_id = core, + .subcore_id = subcore, + }, loc); + defer remote_dma.deinit(); + try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_dma.operand(5).eql(subcore)); +} + +test "matmul and reinterpret cast satisfy current required properties" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); + const lhs = args.addArgument(vector_type, loc); + const rhs = args.addArgument(vector_type, loc); + const acc = args.addArgument(vector_type, loc); + const product = mosaic_tpu.matmul(ctx, lhs, rhs, acc, vector_type, .{}, loc); + defer product.deinit(); + try std.testing.expect(product.verify()); + try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); + + const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); + const ref = args.addArgument(ref_type, loc); + const cast = mosaic_tpu.reinterpret_cast(ctx, ref, ref_type, loc); + defer cast.deinit(); + try expectSegments(cast, &.{ 1, 0, 0, 0 }); + try std.testing.expect(cast.verify()); +} + +test "legacy reduction extrema and scan axis preserve their semantics" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); + const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); + const floats = args.addArgument(float_type, loc); + const integers = args.addArgument(int_type, loc); + const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); + const float_min = mosaic_tpu.all_reduce(ctx, floats, 1, .min, float_type, loc); + defer float_min.deinit(); + const int_max = mosaic_tpu.all_reduce(ctx, integers, 1, .max, int_type, loc); + defer int_max.deinit(); + try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + + const int_scan = mosaic_tpu.scan(ctx, integers, .max, int_type, .{ .mask = mask }, loc); + defer int_scan.deinit(); + try std.testing.expect(int_scan.verify()); + try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); + const float_scan = mosaic_tpu.scan(ctx, floats, .min, float_type, .{}, loc); + defer float_scan.deinit(); + try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); +} + +test "tpu types are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const semaphore = try mosaic_tpu.SemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.SemaphoreType) != null); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) == null); + try expectPrints("!tpu.semaphore", semaphore); + + const dma_semaphore = try mosaic_tpu.DMASemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.dmaSemaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) != null); + try expectPrints("!tpu.dma_semaphore", dma_semaphore); +} + +test "tpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + try expectEnumAttr(mosaic_tpu.ReductionKindAttr, ctx); + try expectEnumAttr(mosaic_tpu.ContractPrecisionAttr, ctx); + try expectEnumAttr(mosaic_tpu.RoundingModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.CoreTypeAttr, ctx); + try expectEnumAttr(mosaic_tpu.DimensionSemanticsAttr, ctx); + try expectEnumAttr(mosaic_tpu.PipelineModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.RevisitModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.MemorySpaceAttr, ctx); + + const smem_tc = try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); + try std.testing.expectEqual(mosaic_tpu.MemorySpace.smem, smem_tc.value()); + try std.testing.expectEqual(mosaic_tpu.CoreType.sc_scalar_subcore, smem_tc.coreType().?); + try expectPrints("#tpu.memory_space", smem_tc); + try std.testing.expectEqual(null, (try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm })).coreType()); + // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); +} + +test "tpu array attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const window = try mosaic_tpu.ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); + try std.testing.expectEqual(2, window.numPadLow()); + try std.testing.expectEqual(2, window.padLow(1)); + try std.testing.expectEqual(2, window.numPadHigh()); + try std.testing.expectEqual(3, window.padHigh(1)); + try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); + try std.testing.expect(mosaic_tpu.elementWindowAttr(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); + + const dims = mosaic_tpu.dotDimensionNumbersAttr(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); + try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); + + const batched = try mosaic_tpu.DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = &.{2}, + .rhsContractingDims = &.{1}, + .lhsNonContractingDims = &.{1}, + .outputDimOrder = &.{ 0, 0, 0, 1 }, + .lhsBatchDims = &.{0}, + .rhsBatchDims = &.{0}, + }); + try std.testing.expectEqual(0, batched.numRhsNonContractingDims()); + try std.testing.expectEqual(4, batched.numOutputDimOrder()); + try std.testing.expectEqual(1, batched.outputDimOrder(3)); + try std.testing.expectEqual(0, batched.rhsBatchDims(0)); + try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); +} + +test "kernel attributes carry their tpu names" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const core_type = mosaic_tpu.coreTypeNamedAttr(ctx, .sc_vector_subcore); + try std.testing.expectEqualStrings("tpu.core_type", core_type.name().str()); + try std.testing.expect(core_type.attribute().eql(mosaic_tpu.CoreType.sc_vector_subcore.attribute(ctx))); + const semantics = mosaic_tpu.dimensionSemanticsNamedAttr(ctx, .arbitrary); + try std.testing.expectEqualStrings("tpu.dimension_semantics", semantics.name().str()); + try std.testing.expect(semantics.attribute().eql(mosaic_tpu.DimensionSemantics.arbitrary.attribute(ctx))); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/nvvm/BUILD.bazel b/mlir/dialects/nvvm/BUILD.bazel new file mode 100644 index 00000000..f6529062 --- /dev/null +++ b/mlir/dialects/nvvm/BUILD.bazel @@ -0,0 +1,168 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "nvvm", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/nvvm", + main = "nvvm.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":nvvm", + "//mlir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "NVVMAttrs.td", + "NVVMDialect.td", + "NVVMEnums.td", + "NVVMOps.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=nvvm", + ], + "NVVMDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=nvvm", + ], + "NVVMDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "NVVMEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "NVVMEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "NVVMOps.h.inc", + ), + ( + ["-gen-op-defs"], + "NVVMOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["NVVMDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "NVVMDialect.cpp", + "NVVMInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/nvvm/NVVMAttrs.td b/mlir/dialects/nvvm/NVVMAttrs.td new file mode 100644 index 00000000..20c3fa51 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMAttrs.td @@ -0,0 +1,52 @@ +//===- NVVMAttrs.td - NVVM attributes of the CuTe DSL ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// The enum attributes are NVVMEnums.td. These are the compiler's other nvvm +// attributes: every mnemonic its parser knows is one of the two. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMATTRS_TD +#define MLIR_DIALECTS_NVVM_NVVMATTRS_TD + +include "mlir/dialects/nvvm/NVVMDialect.td" +include "mlir/dialects/nvvm/NVVMEnums.td" + +def NVVM_MMAShapeAttr : NVVM_Attr<"MMAShape", "shape"> { + let summary = "Shape of an MMA operation"; + let description = "`#nvvm.shape`"; + let parameters = (ins "int":$m, "int":$n, "int":$k); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def NVVM_LdStMatrixShapeAttr : NVVM_Attr<"LdStMatrixShape", "ld_st_matrix_shape"> { + let summary = "Matrix shape of ldmatrix, stmatrix and movmatrix"; + let description = "`#nvvm.ld_st_matrix_shape`"; + let parameters = (ins "int":$m, "int":$n); + let assemblyFormat = "`<` struct(params) `>`"; +} + +// Defaulted fields print only when set; the defaults and the verifier's rules +// and messages are the compiler's. +def NVVM_TargetAttr : NVVM_Attr<"Target", "target"> { + let summary = "GPU target of an NVVM module"; + let description = [{ + `#nvvm.target` + }]; + let parameters = (ins + DefaultValuedParameter<"int", "2">:$O, + StringRefParameter<"target triple", "\"nvptx64-nvidia-cuda\"">:$triple, + StringRefParameter<"target chip", "\"sm_75\"">:$chip, + StringRefParameter<"target features", "\"\"">:$features, + OptionalParameter<"::mlir::DictionaryAttr">:$flags, + OptionalParameter<"::mlir::ArrayAttr">:$link, + DefaultValuedParameter<"bool", "false">:$verifyTarget + ); + let assemblyFormat = "(`<` struct(params)^ `>`)?"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMATTRS_TD diff --git a/mlir/dialects/nvvm/NVVMDialect.cpp b/mlir/dialects/nvvm/NVVMDialect.cpp new file mode 100644 index 00000000..518e825d --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.cpp @@ -0,0 +1,51 @@ +//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL ------------===// + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::nvvm; + +#include "mlir/dialects/nvvm/NVVMDialect.cpp.inc" + +#include "mlir/dialects/nvvm/NVVMEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" + +void NVVMDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" + >(); +} + +// The compiler's rules and messages. +LogicalResult TargetAttr::verify(function_ref emitError, + int O, StringRef triple, StringRef chip, + StringRef, DictionaryAttr, ArrayAttr link, + bool) { + if (O < 0 || O > 3) + return emitError() + << "The optimization level must be a number between 0 and 3."; + if (triple.empty()) + return emitError() << "The target triple cannot be empty."; + if (chip.empty()) + return emitError() << "The target chip cannot be empty."; + if (link && !llvm::all_of(link, [](Attribute attr) { + return attr && isa(attr); + })) + return emitError() + << "All the elements in the `link` array must be strings."; + return success(); +} diff --git a/mlir/dialects/nvvm/NVVMDialect.h b/mlir/dialects/nvvm/NVVMDialect.h new file mode 100644 index 00000000..daa1c67e --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.h @@ -0,0 +1,27 @@ +//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ----*- C++ -*-===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_H +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mlir/dialects/nvvm/NVVMDialect.h.inc" + +#include "mlir/dialects/nvvm/NVVMEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/nvvm/NVVMAttrs.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/nvvm/NVVMOps.h.inc" + +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_H diff --git a/mlir/dialects/nvvm/NVVMDialect.td b/mlir/dialects/nvvm/NVVMDialect.td new file mode 100644 index 00000000..03ce7682 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.td @@ -0,0 +1,33 @@ +//===- NVVMDialect.td - NVVM dialect of the CuTe DSL -------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_TD +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def NVVM_Dialect : Dialect { + let name = "nvvm"; + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let summary = "NVIDIA's NVVM dialect as the CuTe-DSL compiler registers it"; + let description = [{ + This dialect mirrors the operation, attribute and enum registry of the + `nvvm` dialect embedded in the nvidia-cutlass-dsl 4.8.0 compiler, built on + NVIDIA's own LLVM revision (23.0.0git 271b151610907). It differs from the + upstream NVVM dialect (`mlir::NVVM`) and is not meant to be registered + next to it: both are named `nvvm`. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; +} + +class NVVM_Op traits = []> + : Op; + +class NVVM_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_TD diff --git a/mlir/dialects/nvvm/NVVMEnums.td b/mlir/dialects/nvvm/NVVMEnums.td new file mode 100644 index 00000000..fbff0bdc --- /dev/null +++ b/mlir/dialects/nvvm/NVVMEnums.td @@ -0,0 +1,1386 @@ +// The nvvm dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_nvvm_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef MLIR_DIALECTS_NVVM_NVVMENUMS_TD +#define MLIR_DIALECTS_NVVM_NVVMENUMS_TD + +include "mlir/dialects/nvvm/NVVMDialect.td" +include "mlir/IR/EnumAttr.td" + +def NVVM_AtomicOpKindEnum : I32EnumAttr<"AtomicOpKind", "operations supported by atom instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"CAS", 3, "cas">, + I32EnumAttrCase<"EXCH", 4, "exch">, + I32EnumAttrCase<"ADD", 5, "add">, + I32EnumAttrCase<"INC", 6, "inc">, + I32EnumAttrCase<"DEC", 7, "dec">, + I32EnumAttrCase<"MIN", 8, "min">, + I32EnumAttrCase<"MAX", 9, "max">, + I32EnumAttrCase<"UMIN", 10, "umin">, + I32EnumAttrCase<"UMAX", 11, "umax">, + I32EnumAttrCase<"FADD", 12, "fadd">, + I32EnumAttrCase<"SUB", 13, "sub"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReductionEnum : I32EnumAttr<"BarrierReduction", "NVVM barrier reduction operation", [ + I32EnumAttrCase<"POPC", 0, "popc">, + I32EnumAttrCase<"AND", 1, "and">, + I32EnumAttrCase<"OR", 2, "or"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReduxKindEnum : I32EnumAttr<"BarrierReduxKind", "NVVM barrier redux kind", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"POPC", 2, "popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BlockScaleFormatEnum : I32EnumAttr<"BlockScaleFormat", "MMA Block Scale Format", [ + I32EnumAttrCase<"UE8M0", 0, "ue8m0">, + I32EnumAttrCase<"UE4M3", 1, "ue4m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CTAGroupKindEnum : I32EnumAttr<"CTAGroupKind", "NVVM CTA group kind", [ + I32EnumAttrCase<"CTA_1", 0, "cta_1">, + I32EnumAttrCase<"CTA_2", 1, "cta_2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CVTPackFloatKindEnum : I32EnumAttr<"CVTPackFloatKind", "NVVM CVT Pack Float kind", [ + I32EnumAttrCase<"F32", 0, "f32">, + I32EnumAttrCase<"F16x2", 1, "f16x2">, + I32EnumAttrCase<"E4M3x2", 2, "e4m3x2">, + I32EnumAttrCase<"E5M2x2", 3, "e5m2x2">, + I32EnumAttrCase<"BF16x2", 4, "bf16x2">, + I32EnumAttrCase<"E2M1x2", 5, "e2m1x2">, + I32EnumAttrCase<"E2M3x2", 6, "e2m3x2">, + I32EnumAttrCase<"E3M2x2", 7, "e3m2x2">, + I32EnumAttrCase<"UE8M0x2", 8, "ue8m0x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "NVVM Cache Eviction Priority", [ + I32EnumAttrCase<"EvictNormal", 0, "evict_normal">, + I32EnumAttrCase<"EvictFirst", 1, "evict_first">, + I32EnumAttrCase<"EvictLast", 2, "evict_last">, + I32EnumAttrCase<"EvictUnchanged", 3, "evict_unchanged">, + I32EnumAttrCase<"NoAllocate", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ClusterLaunchControlQueryTypeEnum : I32EnumAttr<"ClusterLaunchControlQueryType", "NVVM ClusterLaunchControlQueryType", [ + I32EnumAttrCase<"IS_CANCELED", 0, "is_canceled">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_X", 1, "get_first_cta_id_x">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Y", 2, "get_first_cta_id_y">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Z", 3, "get_first_cta_id_z"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CompareOpKindEnum : I32EnumAttr<"CompareOpKind", "Comparison operator encoding", [ + I32EnumAttrCase<"EQ", 0, "eq">, + I32EnumAttrCase<"NE", 1, "ne">, + I32EnumAttrCase<"UGT", 2, "ugt">, + I32EnumAttrCase<"UGE", 3, "uge">, + I32EnumAttrCase<"ULT", 4, "ult">, + I32EnumAttrCase<"ULE", 5, "ule">, + I32EnumAttrCase<"SGT", 6, "sgt">, + I32EnumAttrCase<"SGE", 7, "sge">, + I32EnumAttrCase<"SLT", 8, "slt">, + I32EnumAttrCase<"SLE", 9, "sle"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP4TypeEnum : I32EnumAttr<"ConvertFP4Type", "NVVM ConvertFP4Type kind", [ + I32EnumAttrCase<"E2M1", 0, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP8TypeEnum : I32EnumAttr<"ConvertFP8Type", "NVVM ConvertFP8Type kind", [ + I32EnumAttrCase<"E4M3", 0, "e4m3">, + I32EnumAttrCase<"E5M2", 1, "e5m2">, + I32EnumAttrCase<"UE8M0", 2, "ue8m0">, + I32EnumAttrCase<"UE5M3", 3, "ue5m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertScaleKindEnum : I32EnumAttr<"ConvertScaleKind", "NVVM ConvertScale kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"UE8M0", 1, "ue8m0"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_DotAccumulateTypeEnum : I32EnumAttr<"DotAccumulateType", "NVVM DotAccumulateType", [ + I32EnumAttrCase<"SIGNED", 1, "signed">, + I32EnumAttrCase<"UNSIGNED", 0, "unsigned"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_EvictKindEnum : I32EnumAttr<"EvictKind", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"FIRST", 1, "evict_first">, + I32EnumAttrCase<"LAST", 2, "evict_last">, + I32EnumAttrCase<"UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NOALLOCATE", 4, "evict_no_allocate">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_FPRoundingModeEnum : I32EnumAttr<"FPRoundingMode", "NVVM FPRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RN", 1, "rn">, + I32EnumAttrCase<"RM", 2, "rm">, + I32EnumAttrCase<"RP", 3, "rp">, + I32EnumAttrCase<"RZ", 4, "rz">, + I32EnumAttrCase<"RNA", 5, "rna">, + I32EnumAttrCase<"RS", 6, "rs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_GridDepActionKindEnum : I32EnumAttr<"GridDepActionKind", "Action kind for grid dependency control", [ + I32EnumAttrCase<"wait", 0, "wait">, + I32EnumAttrCase<"launch_dependents", 1, "launch_dependents"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_IntegerRoundingModeEnum : I32EnumAttr<"IntegerRoundingMode", "NVVM IntegerRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RNI", 1, "rni">, + I32EnumAttrCase<"RZI", 2, "rzi">, + I32EnumAttrCase<"RMI", 3, "rmi">, + I32EnumAttrCase<"RPI", 4, "rpi"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LdStMatrixEltTypeEnum : I32EnumAttr<"LdStMatrixEltType", "Element type for ldmatrix and stmatrix", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B8", 1, "b8">, + I32EnumAttrCase<"B8X16_B6X16_P32", 2, "b8x16.b6x16_p32">, + I32EnumAttrCase<"B8X16_B4X16_P64", 3, "b8x16.b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierExtKindEnum : I32EnumAttr<"LoadCacheModifierExtKind", "NVVM load cache modifier kind(Ext)", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierKindEnum : I32EnumAttr<"LoadCacheModifierKind", "NVVM load cache modifier kind", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadShapeEnum : I32EnumAttr<"LoadShape", "shape attribute for ldmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M8N16", 1, "m8n16">, + I32EnumAttrCase<"M16N16", 3, "m16n16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadSrcFormatEnum : I32EnumAttr<"LoadSrcFormat", "source format for ldmatrix", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64">, + I32EnumAttrCase<"B8", 2, "b8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierLayoutEnum : I32EnumAttr<"MBarrierLayout", "NVVM MBarrier Layout", [ + I32EnumAttrCase<"V0", 0, "v0">, + I32EnumAttrCase<"V1", 1, "v1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierPhaseEnum : I32EnumAttr<"MBarrierPhase", "NVVM mbarrier phase type", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PRIMARY", 1, "primary">, + I32EnumAttrCase<"CONDITIONAL", 2, "conditional"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierScopeKindEnum : I32EnumAttr<"MBarrierScopeKind", "NVVM MBarrier scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierSpaceKindEnum : I32EnumAttr<"MBarrierSpaceKind", "NVVM MBarrier space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierTxnKindEnum : I32EnumAttr<"MBarrierTxnKind", "NVVM MBarrier Transaction kind", [ + I32EnumAttrCase<"ARRIVE", 0, "arrive">, + I32EnumAttrCase<"ARRIVE_DROP", 1, "arrive_drop">, + I32EnumAttrCase<"ARRIVE_EXPECT_TX", 2, "arrive_expect_tx">, + I32EnumAttrCase<"ARRIVE_DROP_EXPECT_TX", 3, "arrive_drop_expect_tx">, + I32EnumAttrCase<"EXPECT_TX", 4, "expect_tx">, + I32EnumAttrCase<"COMPLETE_TX", 5, "complete_tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierWaitKindEnum : I32EnumAttr<"MBarrierWaitKind", "NVVM MBarrier wait kind", [ + I32EnumAttrCase<"TEST", 0, "test">, + I32EnumAttrCase<"TRY", 1, "try"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAB1OpEnum : I32EnumAttr<"MMAB1Op", "MMA binary operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMABlockScaleKindEnum : I32EnumAttr<"MMABlockScaleKind", "Block Scale Kind", [ + I32EnumAttrCase<"MXF8F6F4", 0, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 1, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 2, "mxf4nvf4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMACtaCountEnum : I32EnumAttr<"MMACtaCount", "MMA CTA count", [ + I32EnumAttrCase<"cta1", 1, "cta1">, + I32EnumAttrCase<"cta2", 2, "cta2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAFragEnum : I32EnumAttr<"MMAFrag", "NVVM MMA frag type", [ + I32EnumAttrCase<"a", 0, "a">, + I32EnumAttrCase<"b", 1, "b">, + I32EnumAttrCase<"c", 2, "c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAKindEnum : I32EnumAttr<"MMAKind", "MMA operation kind", [ + I32EnumAttrCase<"f8f6f4", 0, "f8f6f4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMALayoutEnum : I32EnumAttr<"MMALayout", "NVVM MMA layout", [ + I32EnumAttrCase<"row", 0, "row">, + I32EnumAttrCase<"col", 1, "col"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMATypesEnum : I32EnumAttr<"MMATypes", "NVVM MMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"f32", 1, "f32">, + I32EnumAttrCase<"tf32", 2, "tf32">, + I32EnumAttrCase<"bf16", 9, "bf16">, + I32EnumAttrCase<"s8", 4, "s8">, + I32EnumAttrCase<"u8", 3, "u8">, + I32EnumAttrCase<"s32", 5, "s32">, + I32EnumAttrCase<"s4", 8, "s4">, + I32EnumAttrCase<"u4", 7, "u4">, + I32EnumAttrCase<"b1", 6, "b1">, + I32EnumAttrCase<"f64", 10, "f64">, + I32EnumAttrCase<"e4m3", 11, "e4m3">, + I32EnumAttrCase<"e5m2", 12, "e5m2">, + I32EnumAttrCase<"e3m2", 13, "e3m2">, + I32EnumAttrCase<"e2m3", 14, "e2m3">, + I32EnumAttrCase<"e2m1", 15, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MatchSyncKindEnum : I32EnumAttr<"MatchSyncKind", "NVVM match sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "NVVM Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"VOLATILE", 8, "volatile">, + I32EnumAttrCase<"CONSTANT", 7, "constant"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "NVVM Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MulModeEnum : I32EnumAttr<"MulMode", "multiply mode attribute", [ + I32EnumAttrCase<"HI", 0, "hi">, + I32EnumAttrCase<"LO", 1, "lo">, + I32EnumAttrCase<"WIDE", 2, "wide"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_NVVMMemorySpaceEnum : I32EnumAttr<"NVVMMemorySpace", "NVVM Memory Space", [ + I32EnumAttrCase<"Generic", 0, "generic">, + I32EnumAttrCase<"Global", 1, "global">, + I32EnumAttrCase<"Shared", 3, "shared">, + I32EnumAttrCase<"Constant", 4, "constant">, + I32EnumAttrCase<"Local", 5, "local">, + I32EnumAttrCase<"Tensor", 6, "tensor">, + I32EnumAttrCase<"SharedCluster", 7, "shared_cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PermuteModeEnum : I32EnumAttr<"PermuteMode", "NVVM permute mode", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"F4E", 1, "f4e">, + I32EnumAttrCase<"B4E", 2, "b4e">, + I32EnumAttrCase<"RC8", 3, "rc8">, + I32EnumAttrCase<"ECL", 4, "ecl">, + I32EnumAttrCase<"ECR", 5, "ecr">, + I32EnumAttrCase<"RC16", 6, "rc16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PrefetchCacheLevelEnum : I32EnumAttr<"PrefetchCacheLevel", "NVVM Prefetch Cache Level", [ + I32EnumAttrCase<"L1", 0, "L1">, + I32EnumAttrCase<"L2", 1, "L2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ProxyKindEnum : I32EnumAttr<"ProxyKind", "Proxy kind", [ + I32EnumAttrCase<"alias", 0, "alias">, + I32EnumAttrCase<"async_", 1, "async">, + I32EnumAttrCase<"async_global", 2, "async.global">, + I32EnumAttrCase<"async_shared", 3, "async.shared">, + I32EnumAttrCase<"TENSORMAP", 4, "tensormap">, + I32EnumAttrCase<"GENERIC", 5, "generic"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionKindEnum : I32EnumAttr<"ReductionKind", "NVVM Reduction Kind attribute", [ + I32EnumAttrCase<"ADD", 1, "add">, + I32EnumAttrCase<"AND", 2, "and">, + I32EnumAttrCase<"MAX", 3, "max">, + I32EnumAttrCase<"MIN", 4, "min">, + I32EnumAttrCase<"OR", 5, "or">, + I32EnumAttrCase<"UMAX", 6, "umax">, + I32EnumAttrCase<"UMIN", 7, "umin">, + I32EnumAttrCase<"XOR", 8, "xor">, + I32EnumAttrCase<"FMIN", 9, "fmin">, + I32EnumAttrCase<"FMAX", 10, "fmax"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Ops supported by red instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"ADD", 3, "add">, + I32EnumAttrCase<"INC", 4, "inc">, + I32EnumAttrCase<"DEC", 5, "dec">, + I32EnumAttrCase<"MIN", 6, "min">, + I32EnumAttrCase<"MAX", 7, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionTypeEnum : I32EnumAttr<"ReductionType", "types supported by red instruction", [ + I32EnumAttrCase<"B32", 0, "b32">, + I32EnumAttrCase<"B64", 1, "b64">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"U64", 3, "u64">, + I32EnumAttrCase<"S32", 4, "s32">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F32", 6, "f32">, + I32EnumAttrCase<"F64", 7, "f64">, + I32EnumAttrCase<"F16", 8, "f16">, + I32EnumAttrCase<"F16X2", 9, "f16x2">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"BF16X2", 11, "bf16x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressElemSizeEnum : I32EnumAttr<"SPCompressElemSize", "Sparse tensor compression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressFactorTypeEnum : I32EnumAttr<"SPCompressFactorType", "Sparse tensor compression factor type", [ + I32EnumAttrCase<"SP2TO4", 0, "sp2to4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressIndexSizeEnum : I32EnumAttr<"SPCompressIndexSize", "Sparse tensor compression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressOpKindEnum : I32EnumAttr<"SPCompressOpKind", "spcompress operation kind", [ + I32EnumAttrCase<"MIN", 0, "min">, + I32EnumAttrCase<"MAX", 1, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressRepFactorEnum : I32EnumAttr<"SPCompressRepFactor", "Sparse tensor compression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64">, + I32EnumAttrCase<"X128", 128, "x128"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressElemSizeEnum : I32EnumAttr<"SPDecompressElemSize", "Sparse tensor decompression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressFactorTypeEnum : I32EnumAttr<"SPDecompressFactorType", "Sparse tensor decompression factor type", [ + I32EnumAttrCase<"SP1TO2", 0, "sp1to2">, + I32EnumAttrCase<"SP1TO4", 1, "sp1to4">, + I32EnumAttrCase<"SP1TO8", 2, "sp1to8">, + I32EnumAttrCase<"SP1TO16", 3, "sp1to16">, + I32EnumAttrCase<"SP2TO4", 4, "sp2to4">, + I32EnumAttrCase<"SP2TO8", 5, "sp2to8">, + I32EnumAttrCase<"SP2TO16", 6, "sp2to16">, + I32EnumAttrCase<"SP4TO8", 7, "sp4to8">, + I32EnumAttrCase<"SP4TO16", 8, "sp4to16">, + I32EnumAttrCase<"SP8TO16", 9, "sp8to16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressIndexSizeEnum : I32EnumAttr<"SPDecompressIndexSize", "Sparse tensor decompression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressRepFactorEnum : I32EnumAttr<"SPDecompressRepFactor", "Sparse tensor decompression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeEnum : I32EnumAttr<"SaturationMode", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite">, + I32EnumAttrCase<"SAT", 2, "sat"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeKindEnum : I32EnumAttr<"SaturationModeKind", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ScaleVecSizeEnum : I32EnumAttr<"ScaleVecSize", "MMA Scale Vector Sizes", [ + I32EnumAttrCase<"X1", 0, "x1">, + I32EnumAttrCase<"X2", 1, "x2">, + I32EnumAttrCase<"X4", 2, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SetMaxRegisterActionEnum : I32EnumAttr<"SetMaxRegisterAction", "NVVM set max register action", [ + I32EnumAttrCase<"decrease", 1, "decrease">, + I32EnumAttrCase<"increase", 0, "increase"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "Shared memory space", [ + I32EnumAttrCase<"shared_cta", 0, "cta">, + I32EnumAttrCase<"shared_cluster", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ShflKindEnum : I32EnumAttr<"ShflKind", "NVVM shuffle kind", [ + I32EnumAttrCase<"bfly", 0, "bfly">, + I32EnumAttrCase<"up", 1, "up">, + I32EnumAttrCase<"down", 2, "down">, + I32EnumAttrCase<"idx", 3, "idx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SparsityFormatEnum : I32EnumAttr<"SparsityFormat", "MMA Sparsity Format", [ + I32EnumAttrCase<"THREAD", 0, "thread"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StateSpaceEnum : I32EnumAttr<"StateSpace", "NVVM State Space", [ + I32EnumAttrCase<"GENERIC", 0, "generic">, + I32EnumAttrCase<"GLOBAL", 1, "global">, + I32EnumAttrCase<"SHARED_CTA", 2, "shared_cta">, + I32EnumAttrCase<"SHARED_CLUSTER", 3, "shared_cluster">, + I32EnumAttrCase<"CONSTANT", 4, "constant">, + I32EnumAttrCase<"LOCAL", 5, "local">, + I32EnumAttrCase<"TENSOR", 6, "tensor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreCacheModifierKindEnum : I32EnumAttr<"StoreCacheModifierKind", "NVVM store cache modifier kind", [ + I32EnumAttrCase<"WB", 0, "wb">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"WT", 3, "wt">, + I32EnumAttrCase<"NONE", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreShapeEnum : I32EnumAttr<"StoreShape", "shape attribute for stmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M16N8", 2, "m16n8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TCBarParamEnum : I32EnumAttr<"TCBarParam", "Cluster MMA Barrier Parameter Type", [ + I32EnumAttrCase<"a1t0", 0, "a1t0">, + I32EnumAttrCase<"a0tx", 1, "a0tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMALoadModeEnum : I32EnumAttr<"TMALoadMode", "NVVM TMA Load Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"IM2COL_W", 2, "im2col_w">, + I32EnumAttrCase<"IM2COL_W_128", 3, "im2col_w_128">, + I32EnumAttrCase<"TILE_GATHER4", 4, "tile_gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAReduxKindEnum : I32EnumAttr<"TMAReduxKind", "NVVM TMA redux kind", [ + I32EnumAttrCase<"ADD", 0, "add">, + I32EnumAttrCase<"MAX", 2, "max">, + I32EnumAttrCase<"MIN", 1, "min">, + I32EnumAttrCase<"INC", 3, "inc">, + I32EnumAttrCase<"DEC", 4, "dec">, + I32EnumAttrCase<"AND", 5, "and">, + I32EnumAttrCase<"OR", 6, "or">, + I32EnumAttrCase<"XOR", 7, "xor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAStoreModeEnum : I32EnumAttr<"TMAStoreMode", "NVVM TMA Store Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"TILE_SCATTER4", 2, "tile_scatter4">, + I32EnumAttrCase<"IM2COL_W", 3, "im2col_w"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpMulticastEnum : I32EnumAttr<"Tcgen05CpMulticast", "tcgen05 cp multicast", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"WARPX2_02_13", 1, "warpx2_02_13">, + I32EnumAttrCase<"WARPX2_01_23", 2, "warpx2_01_23">, + I32EnumAttrCase<"WARPX4", 3, "warpx4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpShapeEnum : I32EnumAttr<"Tcgen05CpShape", "tcgen05 cp shapes", [ + I32EnumAttrCase<"SHAPE_128x256b", 0, "shape_128x256b">, + I32EnumAttrCase<"SHAPE_4x256b", 1, "shape_4x256b">, + I32EnumAttrCase<"SHAPE_128x128b", 2, "shape_128x128b">, + I32EnumAttrCase<"SHAPE_64x128b", 3, "shape_64x128b">, + I32EnumAttrCase<"SHAPE_32x128b", 4, "shape_32x128b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpSrcFormatEnum : I32EnumAttr<"Tcgen05CpSrcFormat", "tcgen05 cp source format", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05FenceKindEnum : I32EnumAttr<"Tcgen05FenceKind", "NVVM Tcgen05 fence kind", [ + I32EnumAttrCase<"BEFORE_THREAD_SYNC", 0, "before">, + I32EnumAttrCase<"AFTER_THREAD_SYNC", 1, "after"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05LdStShapeEnum : I32EnumAttr<"Tcgen05LdStShape", "allowed 32-bit signless integer cases: 0, 1, 2, 3, 4", [ + I32EnumAttrCase<"SHAPE_16X64B", 0, "shape_16x64b">, + I32EnumAttrCase<"SHAPE_16X128B", 1, "shape_16x128b">, + I32EnumAttrCase<"SHAPE_16X256B", 2, "shape_16x256b">, + I32EnumAttrCase<"SHAPE_32X32B", 3, "shape_32x32b">, + I32EnumAttrCase<"SHAPE_16X32BX2", 4, "shape_16x32bx2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMABlockScaleEnum : I32EnumAttr<"Tcgen05MMABlockScale", "tcgen05.mma block scale attribute", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"BLOCK16", 1, "block16">, + I32EnumAttrCase<"BLOCK32", 2, "block32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorBBufferEnum : I32EnumAttr<"Tcgen05MMACollectorBBuffer", "tcgen05 MMA Collector Buffer B Attribute", [ + I32EnumAttrCase<"B0", 0, "b0">, + I32EnumAttrCase<"B1", 1, "b1">, + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B3", 3, "b3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorOpEnum : I32EnumAttr<"Tcgen05MMACollectorOp", "tcgen05.mma Collector Buffer Operation", [ + I32EnumAttrCase<"DISCARD", 0, "discard">, + I32EnumAttrCase<"LASTUSE", 1, "lastuse">, + I32EnumAttrCase<"FILL", 2, "fill">, + I32EnumAttrCase<"USE", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMAKindEnum : I32EnumAttr<"Tcgen05MMAKind", "tcgen05 MMA Supported Types", [ + I32EnumAttrCase<"F16", 0, "f16">, + I32EnumAttrCase<"TF32", 1, "tf32">, + I32EnumAttrCase<"F8F6F4", 2, "f8f6f4">, + I32EnumAttrCase<"I8", 3, "i8">, + I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 5, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">, + I32EnumAttrCase<"TI16", 7, "ti16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05WaitKindEnum : I32EnumAttr<"Tcgen05WaitKind", "NVVM Tcgen05 wait kind", [ + I32EnumAttrCase<"LOAD", 0, "load">, + I32EnumAttrCase<"STORE", 1, "store"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapElemtypeEnum : I32EnumAttr<"TensormapElemtype", "NVVM Tensormap Elemtype", [ + I32EnumAttrCase<"U8", 0, "u8">, + I32EnumAttrCase<"U16", 1, "u16">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"S32", 3, "s32">, + I32EnumAttrCase<"U64", 4, "u64">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F16", 6, "f16">, + I32EnumAttrCase<"F32", 7, "f32">, + I32EnumAttrCase<"F32_FTZ", 8, "f32.ftz">, + I32EnumAttrCase<"F64", 9, "f64">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"TF32", 11, "tf32">, + I32EnumAttrCase<"TF32_FTZ", 12, "tf32.ftz">, + I32EnumAttrCase<"B4x16", 13, "b4x16">, + I32EnumAttrCase<"B4x16P64", 14, "b4x16_p64">, + I32EnumAttrCase<"B6x16P32", 15, "b6x16_p32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFieldEnum : I32EnumAttr<"TensormapField", "NVVM Tensormap Field Kind", [ + I32EnumAttrCase<"GLOBAL_ADDRESS", 0, "global_address">, + I32EnumAttrCase<"RANK", 1, "rank">, + I32EnumAttrCase<"BOX_DIM", 2, "box_dim">, + I32EnumAttrCase<"GLOBAL_DIM", 3, "global_dim">, + I32EnumAttrCase<"GLOBAL_STRIDE", 4, "global_stride">, + I32EnumAttrCase<"ELEMENT_STRIDE", 5, "element_stride">, + I32EnumAttrCase<"ELEMTYPE", 6, "elemtype">, + I32EnumAttrCase<"INTERLEAVE_LAYOUT", 7, "interleave_layout">, + I32EnumAttrCase<"SWIZZLE_MODE", 8, "swizzle_mode">, + I32EnumAttrCase<"SWIZZLE_ATOMICITY", 9, "swizzle_atomicity">, + I32EnumAttrCase<"FILL_MODE", 10, "fill_mode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFillModeEnum : I32EnumAttr<"TensormapFillMode", "NVVM Tensormap Fill Mode", [ + I32EnumAttrCase<"ZERO", 0, "zero">, + I32EnumAttrCase<"OOB_NAN", 1, "oob_nan"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapInterleaveLayoutEnum : I32EnumAttr<"TensormapInterleaveLayout", "NVVM Tensormap Interleave Layout", [ + I32EnumAttrCase<"NO_INTERLEAVE", 0, "no_interleave">, + I32EnumAttrCase<"B16", 1, "b16">, + I32EnumAttrCase<"B32", 2, "b32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleAtomicityEnum : I32EnumAttr<"TensormapSwizzleAtomicity", "NVVM Tensormap Swizzle Atomicity", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B32_FLIP_8B", 2, "b32_flip_b8">, + I32EnumAttrCase<"B64", 3, "b64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleModeEnum : I32EnumAttr<"TensormapSwizzleMode", "NVVM Tensormap Swizzle Mode", [ + I32EnumAttrCase<"NO_SWIZZLING", 0, "no_swizzling">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B64", 2, "b64">, + I32EnumAttrCase<"B128", 3, "b128">, + I32EnumAttrCase<"B96", 4, "b96"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TmemLayoutEnum : I32EnumAttr<"TmemLayout", "Tensor Memory Layout Enumerated Type", [ + I32EnumAttrCase<"tmem_16dp_128bit", 0, "tmem_16dp_128bit">, + I32EnumAttrCase<"tmem_16dp_256bit", 1, "tmem_16dp_256bit">, + I32EnumAttrCase<"tmem_32dp_32bit", 2, "tmem_32dp_32bit">, + I32EnumAttrCase<"tmem_16dp_64bit", 3, "tmem_16dp_64bit">, + I32EnumAttrCase<"tmem_16dp_32bit_t0_t15", 4, "tmem_16dp_32bit_t0_t15">, + I32EnumAttrCase<"tmem_16dp_32bit_t16_t31", 5, "tmem_16dp_32bit_t16_t31"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ValidatePatternEnum : I32EnumAttr<"ValidatePattern", "NVVM validate data pattern", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PER_16BYTES_80000000", 1, "per_16bytes_80000000">, + I32EnumAttrCase<"PER_16BYTES_8000", 2, "per_16bytes_8000">, + I32EnumAttrCase<"PER_16BYTES_80", 3, "per_16bytes_80">, + I32EnumAttrCase<"PER_16BYTES_8", 4, "per_16bytes_8">, + I32EnumAttrCase<"PER_ELEMENT_FF", 5, "per_element_ff">, + I32EnumAttrCase<"PER_ELEMENT_80000000", 6, "per_element_80000000">, + I32EnumAttrCase<"PER_ELEMENT_8000", 7, "per_element_8000">, + I32EnumAttrCase<"PER_ELEMENT_80", 8, "per_element_80">, + I32EnumAttrCase<"PER_ELEMENT_8", 9, "per_element_8">, + I32EnumAttrCase<"PER_16BYTES_80000000_PAIRED", 10, "per_16bytes_80000000_paired"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_VoteSyncKindEnum : I32EnumAttr<"VoteSyncKind", "NVVM vote sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all">, + I32EnumAttrCase<"ballot", 2, "ballot">, + I32EnumAttrCase<"uni", 3, "uni"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleInEnum : I32EnumAttr<"WGMMAScaleIn", "WGMMA overflow options", [ + I32EnumAttrCase<"one", 1, "one">, + I32EnumAttrCase<"neg", 2, "neg"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleOutEnum : I32EnumAttr<"WGMMAScaleOut", "WGMMA input predicate", [ + I32EnumAttrCase<"zero", 0, "zero">, + I32EnumAttrCase<"one", 1, "one"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMATypesEnum : I32EnumAttr<"WGMMATypes", "NVVM WGMMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"tf32", 1, "tf32">, + I32EnumAttrCase<"u8", 2, "u8">, + I32EnumAttrCase<"s8", 3, "s8">, + I32EnumAttrCase<"b1", 4, "b1">, + I32EnumAttrCase<"bf16", 5, "bf16">, + I32EnumAttrCase<"e4m3", 6, "e4m3">, + I32EnumAttrCase<"e5m2", 7, "e5m2">, + I32EnumAttrCase<"f32", 8, "f32">, + I32EnumAttrCase<"s32", 9, "s32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_AtomicOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_BarrierReductionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BarrierReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BlockScaleFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CTAGroupKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CVTPackFloatKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CacheEvictionPriorityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ClusterLaunchControlQueryTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_CompareOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP4TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP8TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_DotAccumulateTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_EvictKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_FPRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_GridDepActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_IntegerRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_L2PrefetchSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LdStMatrixEltTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_LoadCacheModifierAttrExt : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierSpaceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierTxnKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierWaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAB1OpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMABlockScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMACtaCountAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAFragAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMALayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MatchSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_MemOrderKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MemScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MulModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_NVVMMemorySpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_NVVM_MBarrierPhaseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PermuteModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PrefetchCacheLevelAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ProxyKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ScaleVecSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SetMaxRegisterActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SharedSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ShflKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SparsityFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StateSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TCBarParamEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMALoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpMulticastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05FenceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05LdStShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMABlockScaleAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorBBufferAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05WaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapElemtypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapFieldAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_TensormapFillModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapInterleaveLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleAtomicityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TmemLayoutEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ValidatePatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_VoteSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_WGMMAScaleInAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMAScaleOutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMENUMS_TD diff --git a/mlir/dialects/nvvm/NVVMInference.cpp b/mlir/dialects/nvvm/NVVMInference.cpp new file mode 100644 index 00000000..78a9e56f --- /dev/null +++ b/mlir/dialects/nvvm/NVVMInference.cpp @@ -0,0 +1,253 @@ +//===- NVVMInference.cpp - result types the compiler infers ---------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::nvvm; + +LogicalResult AddFOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +AddPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +AtomicRMWOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +BarrierOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +CompareAndSetOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult FAbsOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +FMAPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult FmaOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +LdMatrixOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierArriveOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTestWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTryWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MatchSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MulOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MulPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ReduxOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult RsqrtOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult ShflOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult SubFOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +SubPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +VoteSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/nvvm/NVVMOps.td b/mlir/dialects/nvvm/NVVMOps.td new file mode 100644 index 00000000..d77563d2 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMOps.td @@ -0,0 +1,6445 @@ +// The nvvm dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_nvvm_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// nvvm.add.packed.f32x2 +// nvvm.atomicrmw +// nvvm.barrier +// nvvm.clusterlaunchcontrol.query.cancel +// nvvm.fma +// nvvm.fma.packed.f32x2 +// nvvm.ldmatrix +// nvvm.match.sync +// nvvm.mbarrier.arrive +// nvvm.mbarrier.arrive.expect_tx +// nvvm.mbarrier.arrive_drop +// nvvm.mbarrier.arrive_drop.expect_tx +// nvvm.mbarrier.test.wait +// nvvm.mbarrier.try_wait +// nvvm.mul (`mode = wide` doubles the operand width) +// nvvm.mul.packed.f32x2 +// nvvm.redux.sync +// nvvm.shfl.sync (`return_value_and_is_valid` returns `!llvm.struct<(T, i1)>`) +// nvvm.sub.packed.f32x2 +// nvvm.vote.sync +// +// Constraints not recovered: +// nvvm.convert.bf16x2.to.f8x2 `dstTy`: `fnv8E5M3FNU` has no type here and is not accepted + +#ifndef MLIR_DIALECTS_NVVM_NVVMOPS_TD +#define MLIR_DIALECTS_NVVM_NVVMOPS_TD + +include "mlir/dialects/nvvm/NVVMAttrs.td" +include "mlir/dialects/nvvm/NVVMEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def NVVM_AnyType : Type, "any type">; +def NVVM_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 4/8">; +def NVVM_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 4 or fixed-length vector of 16-bit signless integer values of length 2">; +def NVVM_FloatingPointOrFixedLengthVector : Type($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "floating-point or fixed-length vector of floating-point values of length 2">; +def NVVM_LLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "LLVM dialect-compatible type">; +def NVVM_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def NVVM_LLVMPointerInAddressSpace02 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace03 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace04 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace05 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 1 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace1 : Type($_self)">, "LLVM pointer in address space 1">; +def NVVM_LLVMPointerInAddressSpace12 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace13 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 5 or LLVM pointer in address space 0 or LLVM pointer in address space 4">; +def NVVM_LLVMPointerInAddressSpace14 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace3 : Type($_self)">, "LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace32 : Type($_self)">, "LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace6 : Type($_self)">, "LLVM pointer in address space 6">; +def NVVM_LLVMPointerInAddressSpace62 : Type($_self)">, "LLVM pointer in address space 6 or 64-bit signless integer">; +def NVVM_LLVMPointerInAddressSpace7 : Type($_self)">, "LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace72 : Type($_self)">, "LLVM pointer in address space 7 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def NVVM_LLVMPointerTypeOrLLVMPointer : Type($_self)">, "LLVM pointer type or LLVM pointer in address space 3">; +def NVVM_LLVMStructureType : Type($_self)">, "LLVM structure type">; +def NVVM_LLVMStructureTypeOr64Bit : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "LLVM structure type or 64-bit float">; +def NVVM_LLVMTypeWithSize : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self)">, "LLVM type with size">; +def NVVM_T128BitSignlessInteger : Type, "128-bit signless integer">; +def NVVM_T16BitFloatOrBfloat16Type : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit float or bfloat16 type or 32-bit float or 64-bit float or vector of 16-bit float or bfloat16 type or 32-bit float or 64-bit float values of length 2">; +def NVVM_T16BitSignlessInteger : Type, "16-bit signless integer">; +def NVVM_T16BitSignlessInteger2 : Type, "16-bit signless integer">, BuildableType<"$_builder.getIntegerType(16)">; +def NVVM_T16BitSignlessIntegerOr32 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 128-bit signless integer or 16-bit float or bfloat16 type or 32-bit float or 64-bit float or 128-bit float or vector of 32-bit float values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T16BitSignlessIntegerOr322 : Type, "16-bit signless integer or 32-bit signless integer">; +def NVVM_T16BitSignlessIntegerOr323 : Type, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer">; +def NVVM_T16BitSignlessIntegerOrVector : Type($_self)">, "16-bit signless integer or vector of 8-bit signless integer values of length 2">; +def NVVM_T1BitSignlessInteger : Type, "1-bit signless integer">; +def NVVM_T1BitSignlessInteger2 : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def NVVM_T1BitSignlessIntegerOr32 : Type, "1-bit signless integer or 32-bit signless integer">; +def NVVM_T1BitSignlessIntegerOrLLVM : Type($_self)">, "1-bit signless integer or LLVM structure type">; +def NVVM_T32BitFloat : Type($_self)">, "32-bit float">; +def NVVM_T32BitFloat2 : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def NVVM_T32BitFloatOr64Bit : Type($_self) || ::llvm::isa<::mlir::Float64Type>($_self)">, "32-bit float or 64-bit float">; +def NVVM_T32BitSignlessInteger : Type, "32-bit signless integer">; +def NVVM_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def NVVM_T32BitSignlessIntegerOr1 : Type, "32-bit signless integer or 1-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr32 : Type($_self)">, "32-bit signless integer or 32-bit float">; +def NVVM_T32BitSignlessIntegerOr322 : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "32-bit signless integer or 32-bit float or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOr64 : Type, "32-bit signless integer or 64-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr642 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "32-bit signless integer or 64-bit signless integer or 64-bit float or bfloat16 type or 16-bit float or 32-bit float or vector of 32-bit float or 32-bit signless integer values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T32BitSignlessIntegerOr643 : Type($_self)">, "32-bit signless integer or 64-bit signless integer or vector of 32-bit signless integer or 64-bit signless integer values of length 2/4">; +def NVVM_T32BitSignlessIntegerOrLLVM : Type($_self)">, "32-bit signless integer or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOrVector : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_T32BitSignlessIntegerOrVector2 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_T32BitSignlessIntegerOrVector3 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4">; +def NVVM_T64BitSignlessInteger : Type, "64-bit signless integer">; +def NVVM_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def NVVM_T64BitSignlessIntegerOr32 : Type, "64-bit signless integer or 32-bit signless integer">; +def NVVM_T8BitSignlessInteger : Type, "8-bit signless integer">, BuildableType<"$_builder.getIntegerType(8)">; +def NVVM_T8BitSignlessInteger2 : Type, "8-bit signless integer">; +def NVVM_T8BitSignlessIntegerOr32 : Type, "8-bit signless integer or 32-bit signless integer">; +def NVVM_VariadicOf16BitSignlessInteger : Type, "variadic of 16-bit signless integer">; +def NVVM_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def NVVM_VariadicOfLLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "variadic of LLVM dialect-compatible type">; +def NVVM_VectorOf16BitFloatValues : Type($_self)">, "vector of 16-bit float values of length 2">; +def NVVM_VectorOf16BitSignlessInteger : Type($_self)">, "vector of 16-bit signless integer values of length 2">; +def NVVM_VectorOf32BitFloatValues : Type($_self)">, "vector of 32-bit float values of length 2">; +def NVVM_VectorOf32BitFloatValues2 : Type($_self)">, "vector of 32-bit float values">; +def NVVM_VectorOf32BitFloatValues3 : Type($_self)">, "vector of 32-bit float values of length 4">; +def NVVM_VectorOf32BitSignlessInteger : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger2 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 or 32-bit signless integer">; +def NVVM_VectorOf32BitSignlessInteger3 : Type($_self)">, "vector of 32-bit signless integer or 32-bit float values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger4 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_VectorOf8BitSignlessInteger : Type($_self)">, "vector of 8-bit signless integer values of length 4">; +def NVVM_VectorOf8BitSignlessInteger2 : Type($_self)">, "vector of 8-bit signless integer values of length 2">; +def NVVM_VectorOfAnyTypeValues : Type($_self)">, "vector of any type values">; +def NVVM_VectorOfBfloat16TypeValuesOf : Type($_self)">, "vector of bfloat16 type values of length 2">; + +// Attribute constraints the bindings name but do not describe. +def NVVM_ConvertBF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF8x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f8E8M0FNU type or f8E4M3FN type or f8E5M2 type or fnv8E5M3FNU type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_FmaRndAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::FPRoundingMode>{::mlir::cutlass_compiler::nvvm::FPRoundingMode::NONE, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RN, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RM, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RP, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RZ}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr>($_self).getValue())">, "NVVM FPRoundingMode kind whose value is one of {none, rm, rn, rp, rz}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::ACQUIRE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, acquire}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "Ops supported by red instruction whose value is none of {inc, dec}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::INC, ::mlir::cutlass_compiler::nvvm::ReductionOp::DEC, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "ReductionOp attribute"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemStOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_LLVMConstantRangeAttr : Attr($_self)">, "A range of two integers, corresponding to LLVM's ConstantRange"> { + let storageType = "::mlir::LLVM::ConstantRangeAttr"; + let returnType = "::mlir::LLVM::ConstantRangeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_TensormapFieldValueAttr : Attr($_self)">, "NVVM Tensormap Elemtype or NVVM Tensormap Interleave Layout or NVVM Tensormap Swizzle Mode or NVVM Tensormap Swizzle Atomicity or NVVM Tensormap Fill Mode"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (NVVMInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class NVVM_InferredOp traits = []> + : NVVM_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def NVVM_AddPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_AddPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_AddPackedF32x2Op : NVVM_InferredOp<"add.packed.f32x2"> { + let description = [{ + This Op performs addition of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_AddPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddFOp : NVVM_InferredOp<"addf", [Pure]> { + let description = [{ + The `nvvm.addf` operation performs floating point addition of two floating + point operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, and flush-to-zero by the `ftz` attribute. + + For more information, see PTX ISA: + - [floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add) + - [half-precision floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-add) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_ApplyPriorityAsyncBulkOp : NVVM_Op<"applypriority.async.bulk"> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the specified + region of L2 cache that are previously present and marked as evict last. + Other L2 cache lines in the specified region are not affected. + + Arguments: + - `ptr`: Global memory pointer. + - `size`: Address range of length. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk %ptr, %size evict_priority = evict_normal + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$ptr, + NVVM_T32BitSignlessInteger:$size, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOp : NVVM_Op<"applypriority.async.bulk.tensor", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `box/coordinates`: The coordinates of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2w_halo, %im2w_offset] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + Variadic:$coordinates, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOverrideOp : NVVM_Op<"applypriority.async.bulk.tensor.override", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + It supports a way to override the global address of the tensor. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `overrideAdrr`: The global address to override. + - `box/coordinates`: The coordinates of the tensor. + - `tensorSize`: The size of the tensor. + - `lowerStride`: The lower stride of the tensor. + - `upperStride`: The upper stride of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] tensor_size [ %ts0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1] tensor_size [ %ts0, %ts1] lower_stride [ %lstrd0] upper_stride [ %ustrd] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_AtomicRMWOp : NVVM_InferredOp<"atomicrmw"> { + let description = [{ + Atomically loads the original value at location `a` into destination register `res`, + performs a reduction operation with operand `b` and the value in location `a`, and + stores the result of the specified operation at location `a`, overwriting the + original value. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses using `Generic Addressing`. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-atom) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace12:$ptr, + NVVM_T16BitSignlessIntegerOr32:$a, + Optional:$b, + NVVM_AtomicOpKindAttr:$op, + OptionalAttr:$memOrder, + OptionalAttr:$syncscope + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr32:$res + ); +} +def NVVM_SyncWarpOp : NVVM_Op<"bar.warp.sync"> { + let description = [{ + The `nvvm.bar.warp.sync` operation performs barrier synchronization for threads + within a warp. + + This operation causes the executing thread to wait until all threads corresponding + to the `mask` operand have executed a `bar.warp.sync` with the same mask value + before resuming execution. + + The `mask` operand specifies the threads participating in the barrier, where each + bit position corresponds to the thread's lane ID within the warp. Only threads with + their corresponding bit set in the mask participate in the barrier synchronization. + + **Important constraints**: + - The behavior is undefined if the executing thread is not included in the mask + (i.e., the bit corresponding to the thread's lane ID is not set) + - For compute capability sm_6x or below, all threads in the mask must execute + the same `bar.warp.sync` instruction in convergence + + This operation also guarantees memory ordering among participating threads. + Threads within the warp that wish to communicate via memory can store to memory, + execute `bar.warp.sync`, and then safely read values stored by other threads + in the warp. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-warp-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask + ); + let results = (outs); +} +def NVVM_BarrierOp : NVVM_InferredOp<"barrier", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.barrier` operation performs barrier synchronization and communication + within a CTA (Cooperative Thread Array). It causes executing threads to wait for + all non-exited threads participating in the barrier to arrive. + + The operation takes two optional operands: + + - `barrierId`: Specifies a logical barrier resource with value 0 through 15. + Each CTA instance has sixteen barriers numbered 0..15. Defaults to 0 if not specified. + - `numberOfThreads`: Specifies the number of threads participating in the barrier. + When specified, the value must be a multiple of the warp size. If not specified, + all threads in the CTA participate in the barrier. + - `reductionOp`: specifies the reduction operation (`popc`, `and`, `or`). + - `reductionPredicate`: specifies the predicate to be used with the + `reductionOp`. + + The barrier operation guarantees that when the barrier completes, prior memory + accesses requested by participating threads are performed relative to all threads + participating in the barrier. It also ensures that no new memory access is + requested by participating threads before the barrier completes. + + When a barrier completes, the waiting threads are restarted without delay, and + the barrier is reinitialized so that it can be immediately reused. + + This operation generates an aligned barrier, indicating that all threads in the CTA + will execute the same barrier instruction. Behavior is undefined if all threads in the + CTA do not reach this instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + Optional:$numberOfThreads, + Optional:$reductionPredicate, + OptionalAttr:$reductionOp + ); + let results = (outs + Optional:$res + ); +} +def NVVM_BarrierArriveOp : NVVM_Op<"barrier.arrive"> { + let description = [{ + Thread that executes this op announces their arrival at the barrier with + given id and continue their execution. + + The default barrier id is 0 that is similar to `nvvm.barrier` Op. When + `barrierId` is not present, the default barrier id is used. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + NVVM_T32BitSignlessInteger:$numberOfThreads + ); + let results = (outs); +} +def NVVM_BarrierCTAArriveOp : NVVM_Op<"barrier.cta.arrive"> { + let description = [{ + The `nvvm.barrier.cta.arrive` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of threads + participating in the barrier. The `barrier.cta.arrive` does not cause the executing thread to wait for + threads of other participating warps. The `aligned` attribute, when set to 1, generates the .aligned + version of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + NVVM_T32BitSignlessInteger:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_BarrierCTAReduxOp : NVVM_Op<"barrier.cta.red"> { + let description = [{ + The `nvvm.barrier.cta.red` is used to perform a reduction operation across threads in the CTA. + The `pred` from all the participating threads are combined using the specified reduction operator. + AND, OR, POPC (population-count) are available as reduction operators. The `barrierId` specifies + a logical barrier resource and the `threadCount` specifies the number of threads participating + in the barrier. When the `threadCount` is not specified, all threads in the CTA participate in + the reduction operation. The `aligned` attribute, when set to 1, generates the .aligned version + of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T1BitSignlessInteger:$pred, + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + NVVM_BarrierReduxKindAttr:$kind, + OptionalAttr:$aligned + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_BarrierCTASyncOp : NVVM_Op<"barrier.cta.sync"> { + let description = [{ + The `nvvm.barrier.cta.sync` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of + threads participating in the barrier. When the `threadCount` is not specified, all threads in the CTA + participate in the barrier. The `aligned` attribute, when set to 1, generates the .aligned version of + the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_Barrier0Op : NVVM_Op<"barrier0"> { + let description = [{ + The `nvvm.barrier0` operation is a convenience operation that performs barrier + synchronization and communication within a CTA (Cooperative Thread Array) using + barrier ID 0. It is functionally equivalent to `nvvm.barrier` or `nvvm.barrier id=0`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_Breakpoint : NVVM_Op<"breakpoint"> { + let description = [{ + Breakpoint suspends execution of the program for debugging. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-brkpt) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_ClmadOp : NVVM_Op<"clmad", [Pure]> { + let description = [{ + Performs a carryless multiplication of `a` and `b`, followed by a carryless + addition of `c`. + When `hi` is true, the result is the upper 64 bits of the product, with the + addition of `c`. Otherwise, the result is the lower 64 bits of the product, + with the addition of `c`. + }]; + let arguments = (ins + NVVM_T64BitSignlessInteger:$a, + NVVM_T64BitSignlessInteger:$b, + NVVM_T64BitSignlessInteger:$c, + OptionalAttr:$high + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_ClusterArriveOp : NVVM_Op<"cluster.arrive"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterArriveRelaxedOp : NVVM_Op<"cluster.arrive.relaxed"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + The .relaxed qualifier on `cluster.arrive` specifies that there are no memory + ordering and visibility guarantees provided for the memory accesses performed prior to + `cluster.arrive`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterWaitOp : NVVM_Op<"cluster.wait"> { + let description = [{ + The `cluster.wait` causes the executing thread to wait for all non-exited threads + of the cluster to perform `cluster.arrive`. The `aligned` attribute, when provided, + generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterLaunchControlQueryCancelOp : NVVM_InferredOp<"clusterlaunchcontrol.query.cancel"> { + let description = [{ + `clusterlaunchcontrol.query.cancel` queries the response of a + `clusterlaunchcontrol.try.cancel` operation specified by operand + `try_cancel_response`. + + Operand `query_type` specifies the type of query to perform and can be one + of the following: + - `is_canceled` : Returns true if the try cancel request succeeded, + and false otherwise. + - `get_first_cta_id_{x/y/z}` : Returns the x, y, or z coordinate of the + first CTA in the canceled cluster. Behaviour is defined only if the try + cancel request succeeded. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel) + }]; + let arguments = (ins + NVVM_T128BitSignlessInteger:$try_cancel_response, + NVVM_ClusterLaunchControlQueryTypeAttr:$query_type + ); + let results = (outs + NVVM_T1BitSignlessIntegerOr32:$res + ); +} +def NVVM_ClusterLaunchControlTryCancelOp : NVVM_Op<"clusterlaunchcontrol.try.cancel"> { + let description = [{ + `clusterlaunchcontrol.try.cancel` requests atomically canceling the launch + of a cluster that has not started running yet. It asynchronously writes an + opaque response to shared memory indicating whether the operation succeeded + or failed. + + Operand `smemAddress` specifies the naturally aligned address of the + 16-byte wide shared memory location where the request's response is written. + + Operand `mbarrier` specifies the mbarrier object used to track the + completion of the asynchronous operation. + + If `multicast` is specified, the response is asynchronously written to the + corresponding local shared memory location (specifed by `addr`) of each CTA + in the requesting cluster. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$smemAddress, + NVVM_LLVMPointerInAddressSpace3:$mbarrier, + UnitAttr:$multicast + ); + let results = (outs); +} +def NVVM_CompareAndSetOp : NVVM_InferredOp<"compare.and.set"> { + let description = [{ + Compare two values and return boolean mask of 1's or 0's. Return value + will be of the same type as the operands (vector<4xi8> or vector<2xi16>). + + Arguments: + - `a`: The first value. + - `b`: The second value. + - `op`: The comparison operator. (eq, ne, ugt, uge, ult, ule, sgt, sge, slt, sle) + + Example: + ```mlir + %res = nvvm.compare.and.set %a, %b : vector<4xi8> + %res = nvvm.compare.and.set %a, %b : vector<2xi16> + ``` + }]; + let arguments = (ins + NVVM_FixedLengthVectorOf8Bit:$a, + NVVM_FixedLengthVectorOf8Bit:$b, + NVVM_CompareOpKindAttr:$op + ); + let results = (outs + NVVM_FixedLengthVectorOf8Bit:$result + ); +} +def NVVM_ConvertPackIntegerOp : NVVM_Op<"convert.and.pack.integer"> { + let description = [{ + This Op converts the given integer inputs to the specified integer type + with saturation and packs the result into the lower bits of the result. + + The result `dst` is represented as an i32 type. + + The optional operand `srcC` must be specified when the converted types add + up to less than 32 bits. The remaining bits of the result are filled with + the lower bits of `srcC`. + + The `convert_type` attribute specifies the integer type to convert to + before packing. The `signed` attribute can be specified to indicate that + the `convert_type` is signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt-pack) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcB, + Optional:$srcC, + UnitAttr:$is_signed, + TypeAttr:$convert_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertBF16x2ToF4x2Op : NVVM_Op<"convert.bf16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertBF16x2ToF6x2Op : NVVM_Op<"convert.bf16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> { + let description = [{ + This Op converts the given bf16 inputs in a bf16x2 vector to the specified + f8 type. The result `dst` is represented either as a packed i16 type or as + a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF8x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToS2F6x2Op : NVVM_Op<"convert.bf16x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in a bf16x2 vector to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF4x2Op : NVVM_Op<"convert.f16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF16x2ToF6x2Op : NVVM_Op<"convert.f16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> { + let description = [{ + This Op converts the given f16 inputs in an f16x2 vector to the specified + f8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values from `a` + are packed such that the value converted from the first element of `a` + is stored in the upper 8 bits of `dst` and the value converted from the + second element of `a` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$a, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToBF16x2Op : NVVM_Op<"convert.f32x2.to.bf16x2"> { + let description = [{ + Converts two F32 values to packed bf16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-bf16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF32x2ToF16x2Op : NVVM_Op<"convert.f32x2.to.f16x2"> { + let description = [{ + Converts two F32 values to packed f16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-f16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp4 type. + + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from `a` is stored in the upper 4 bits + of `dst` and the value converted from `b` is stored in the lower 4 bits of + `dst`. + + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp6 type. + The result `dst` is represented either as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + with 2 MSB bits padded with zeros and the value converted from `b` is + stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes respectively. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given f32 inputs to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x4ToF4x4Op : NVVM_Op<"convert.f32x4.to.f4x4"> { + let description = [{ + Converts a vector<4xf32> to packed f4x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertF32x4ToF6x4Op : NVVM_Op<"convert.f32x4.to.f6x4"> { + let description = [{ + Converts a vector<4xf32> to packed f6x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x4ToF8x4Op : NVVM_Op<"convert.f32x4.to.f8x4"> { + let description = [{ + Converts a vector<4xf32> to packed f8x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF4x2ToF16x2Op : NVVM_Op<"convert.f4x2.to.f16x2"> { + let description = [{ + This Op converts the given f4 inputs in a packed i8 to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF6x2ToF16x2Op : NVVM_Op<"convert.f6x2.to.f16x2"> { + let description = [{ + This Op converts the given f6 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF8x2ToBF16x2Op : NVVM_Op<"convert.f8x2.to.bf16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to bf16. + + The result `dst` is represented as a vector of bf16 elements. + The `sat` attribute specifies the saturation mode. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF8x2ToF16x2Op : NVVM_Op<"convert.f8x2.to.f16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertFloatToIntegerOp : NVVM_Op<"convert.float.to.integer"> { + let description = [{ + This Op converts the given float input to the specified integer type. + + The `rnd` attribute specifies the rounding mode. + The `sat` atttribute, when set, limits the result to the range of the + integer type. + The `ftz` attribute, when set, converts denormal inputs to zero. + The `signed` attribute, when set, converts the input to a signed integer. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + UnitAttr:$sat, + UnitAttr:$ftz, + UnitAttr:$is_signed + ); + let results = (outs + NVVM_T8BitSignlessIntegerOr32:$dst + ); +} +def NVVM_ConvertFloatToTF32Op : NVVM_Op<"convert.float.to.tf32"> { + let description = [{ + This Op converts the given f32 input to tf32. + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ConvertS2F6x2ToBF16x2Op : NVVM_Op<"convert.s2f6x2.to.bf16x2"> { + let description = [{ + This Op converts a pair of s2f6x2 inputs to bf16x2 type. The result `dst` + is represented as a vector of two bf16 elements. + + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_CosOp : NVVM_Op<"cos", [Pure]> { + let description = [{ + Computes a fast approximation of the cosine of the input value (in + radians). The `ftz` attribute, when set, flushes subnormal inputs + and results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group"> { + let description = [{ + This Op commits all prior initiated but uncommitted cp.async.bulk + instructions into a cp.async.bulk-group. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to + global memory. The 32-bit operand `size` specifies the amount of + memory to be copied, in terms of number of bytes. `size` must be a + multiple of 16. The `l2CacheHint` operand is optional, and it is used + to specify cache eviction policy that may be used during the memory + access. The `byteMask` operand is optional. The i-th bit in the 16-bit + wide `byteMask` specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. If the bit is set, + the byte is copied. + + Example: + ```mlir + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size + : !llvm.ptr<1>, !llvm.ptr<3> + + // with l2_cache_hint + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch + : !llvm.ptr<1>, !llvm.ptr<3> + + // with byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + + // with both l2_cache_hint and byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + Optional:$byteMask + ); + let results = (outs); +} +def NVVM_CpAsyncBulkPrefetchOp : NVVM_Op<"cp.async.bulk.prefetch"> { + let description = [{ + Initiates an asynchronous prefetch of data from the location + specified by `srcMem` to the L2 cache. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + The `evict` operand is optional, and it is used to specify cache eviction + priority. The `evict` operand is mutually exclusive with the `l2CacheHint` + operand. + + Example: + ```mlir + nvvm.cp.async.bulk.prefetch %src, %size : !llvm.ptr<1> + + // with l2_cache_hint + nvvm.cp.async.bulk.prefetch %src, %size l2_cache_hint = %ch : !llvm.ptr<1> + ``` + Example: + ```mlir + // with evict_priority + nvvm.cp.async.bulk.prefetch %src, %size evict_priority = evict_last : !llvm.ptr<1> + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from global memory to shared + memory or shared_cluster memory. + + The `multicastMask` operand is optional and can be used only when the + destination is shared::cluster memory. When it is present, this Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$validatePattern + ); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.shared.cta"> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to Shared + cluster memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy of the tensor data from shared::cta + memory to global memory. This Op supports all the store modes specified in + `TMAStoreMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from shared + memory to global memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE`, `IM2COL`, `IM2COL_W`, and `TILE_SCATTER4`) via the `mode` attribute + + Examples: + + // Basic usage with 1D box coordinates (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] + l2_cache_hint = %ch { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOp : NVVM_Op<"cp.async.bulk.tensor.prefetch", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous prefetch operation on the tensor data from global + memory to L2 cache. This Op supports all the load modes specified in + `TMALoadMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.prefetch.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that prefetches data from + global memory to L2 cache overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from global memory location specified by `overrideAdrr` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Can specify im2col offsets via `im2colOffsets` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports an optional cache eviction priority with `evict` (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + + Examples: + + // Basic prefetch operations + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + + // With evict_priority + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size specification + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] tensor_size[%ts0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and stride specifications + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] lower_stride[%ls0] upper_stride[%us0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With im2col offsets + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] + im2col[%off0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3] + im2col[%off0, %off1] : !llvm.ptr, !llvm.ptr<1> + ``` + + // Dim stride with cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3, %d4] + tensor_size[%ts0, %ts1, %ts2, %ts3, %ts4] + lower_stride[%ls0, %ls1, %ls2, %ls3] + upper_stride[%us0] + l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$evict, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOp : NVVM_Op<"cp.async.bulk.tensor.reduce", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous reduction operation of tensor data in + global memory with tensor data in shared memory. + + The `mode` attribute indicates whether the copy mode is tile or im2col. + The `redOp` attribute specifies the reduction operations applied. + The supported reduction operations are: + {add, min, max, inc, dec, and, or, xor} + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.reduce.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that performs a reduction operation + overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE` or `IM2COL`) via the `mode` attribute + - Performs a reduction operation specified by the `redKind` attribute (supports ADD, MIN, MAX, + INC, DEC, AND, OR, XOR) that combines the source data stored at the destination location + + Examples: + + // Override address with ADD reduction (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1] + l2_cache_hint = %ch { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode with XOR reduction + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size (dim variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride information (dim_stride variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation on the tensor data from global + memory to shared::cluster (or) shared::cta memory. This Op supports all + the load modes specified in `TMALoadMode`. + + The `multicastMask` operand is optional. When it is present, the Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + When the `isCTAOnly` attribute is set to true, the destination is + shared::cta only. Hence, `multicastMask` and `CTAGroup` are not applicable + when `isCTAOnly` is true. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + NVVM_LLVMPointerInAddressSpace3:$mbar, + Variadic:$im2colOffsets, + Optional:$multicastMask, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$validatePattern, + OptionalAttr:$mode, + OptionalAttr:$isCTAOnly, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from global + memory to shared::cluster or shared::cta memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Writes data to shared::cluster or shared::cta memory location specified by `dstMem` + - Uses a TMA descriptor `tmaDesc` to specify the properties of the tensor copy operation + - Uses `mbar` to specify the memory barrier for the operation + - Overrides the default source address with `overrideAddr` + - Specifies box coordinates via `coordinates` array + - Specifies im2col arguments via `im2colArgs` (optional) + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports multicast mask via `multicastMask` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + - Supports different group kinds (`CTA_1` or `CTA_2`) via the `group` attribute (optional) + - Supports different validate load data patterns via the `validatePattern` attribute (optional) + + Examples: + // Basic usage with box coordinates + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With multicast mask (i16) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With multicast mask (i32) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc32 { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + + // With both multicast mask and L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With CTA_2 group + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and strides + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + tensor_size[%ts0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL mode with im2col arguments + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%im2col0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W_128 mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Comprehensive example with all optional parameters + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] + multicast_mask = %mc32 + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace72:$dstMem, + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_LLVMPointerInAddressSpace1:$overrideAddr, + Variadic:$coordinates, + Variadic:$im2colArgs, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$group, + OptionalAttr:$validatePattern, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkWaitGroupOp : NVVM_Op<"cp.async.bulk.wait_group"> { + let description = [{ + Op waits for completion of the most recent bulk async-groups. + + The `$group` operand tells waiting has to be done until for $group or fewer + of the most recent bulk async-groups. If `$group` is 0, the op wait until + all the most recent bulk async-groups have completed. + + The `$read` indicates that the waiting has to be done until all the bulk + async operations in the specified bulk async-group have completed reading + from their source locations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group) + }]; + let arguments = (ins + I32Attr:$group, + UnitAttr:$read + ); + let results = (outs); +} +def NVVM_CpAsyncCommitGroupOp : NVVM_Op<"cp.async.commit.group"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncMBarrierArriveOp : NVVM_Op<"cp.async.mbarrier.arrive"> { + let description = [{ + The `cp.async.mbarrier.arrive` Op makes the *mbarrier object* track + all prior cp.async operations initiated by the executing thread. + The `addr` operand specifies the address of the *mbarrier object* + in generic or shared::cta address space. When it is generic, the + underlying memory should fall within the shared::cta space; + otherwise the behavior is undefined. The `noinc` attr impacts + how the mbarrier's state is updated. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + OptionalAttr:$noinc + ); + let results = (outs); +} +def NVVM_CpAsyncOp : NVVM_Op<"cp.async.shared.global"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$dst, + NVVM_LLVMPointerInAddressSpace1:$src, + Optional:$cpSize, + I32Attr:$size, + NVVM_LoadCacheModifierAttr:$modifier + ); + let results = (outs); +} +def NVVM_CpAsyncWaitGroupOp : NVVM_Op<"cp.async.wait.group"> { + let arguments = (ins + I32Attr:$n + ); + let results = (outs); +} +def NVVM_CvtPackFloatOp : NVVM_Op<"cvt.packfloat"> { + let description = [{ + This Op converts the given floating point input (`srcA`) to another + packed floating point type (`res`). The operand `from` represents the + source type and `to` represents the destination type. + The result is merged with the given `srcC`. Both the input and result + are represented as 'i32' type. + + Below table describes the format of the result: + ``` + |--------------|-------------------------------------------------------| + | Dst Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The result contains two fp16 values, in the low | + | bf16x2 | and high bits. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The result contains two fp8 values in the low or | + | e5m2x2 | high 16-bits of `srcC` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the result is packed in | + | | bits `[7:0]` and `[15:8]` of `srcC` and the high | + | | 16-bits of srcC remain unchanged. | + | | When `ExtractHi` is 1, the result is packed in | + | | bits `[23:16]` and `[31:24]` of `srcC` and the low | + | | 16-bits of srcC remain unchanged. | + |--------------|-------------------------------------------------------| + | e2m3x2 | The result contains two 6-bit floating point values | + | e3m2x2 | with the two MSB's padded with zeroes. The 16-bits | + | | of the result are then packed in the same format | + | | described above. | + |--------------|-------------------------------------------------------| + | e2m1x2 | The result contains two 4-bit floating point values | + | | packed in bits `[7:0]` or `[23:16]` of `srcC` | + | | depending on `ExtractHi`. The other bits of `srcC` | + | | remain unchanged. | + |--------------|-------------------------------------------------------| + ``` + + Below table describes the format of the source operand: + ``` + |--------------|-------------------------------------------------------| + | Src Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The two fp16 values are taken from the low and high | + | bf16x2 | high bits of operand `srcA`. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The two fp8 values are taken from the low or high | + | e5m2x2 | 16-bits of `srcA` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the inputs are taken from | + | | bits `[7:0]` and `[15:8]` of `srcA`. | + | | When `ExtractHi` is 1, the inputs are sourced from | + | | bits `[23:16]` and `[31:24]` of `srcA`. | + |--------------|-------------------------------------------------------| + | e2m3x2 | Similar to the row above. | + | e3m2x2 | | + |--------------|-------------------------------------------------------| + | e2m1x2 | The two 4-bit floating point input values are taken | + | | from bits `[7:0]` or `[23:16]` of `srcA` depending | + | | on `ExtractHi`. | + |--------------|-------------------------------------------------------| + ``` + The `relu` attribute, when set, lowers to the '.relu' variant of the + cvt instruction. The `rnd` and `sat` attributes specify the rounding + and saturation respectively. The ue8m0x2, e2m3x2, e3m2x2 and e2m1x2 + types, either as source or destination types, are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$from, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtPackFloatF32Op : NVVM_Op<"cvt.packfloat.f32"> { + let description = [{ + This Op converts two given F32 inputs (in `srcA` and `srcB`) to a + packed floating point type requested in `to`, and the + result is merged with the given `srcC`. + `extractHi`= 0: the result is packed into the low 16-bits of `srcC`. + (and the high-16 bits of `srcC` remain as is) + `extractHi`= 1: the result is packed into the high 16-bits of `srcC`. + (and the low-16 bits of `srcC` remain as is) + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. The ue8m0x2, + e2m3x2, e3m2x2 and e2m1x2 types are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$srcA, + NVVM_T32BitFloat:$srcB, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtToF4x8PackedOp : NVVM_Op<"cvt.to.f4x8.packed"> { + let description = [{ + Fused conversion + packing: performs 4 pairwise conversions to f4 and + packs the 4 byte-sized results into a single i32 using inline PTX + `{ .reg .b8; 4x cvt; mov.b32 }`. + + This avoids the NVPTX backend's lack of a B8 register class, which + causes 3x `prmt.b32` instructions instead of a single `mov.b32` when + packing 4x i8 values via `BUILD_VECTOR v4i8`. + + For f32 source: takes 8 f32 values (4 high-low pairs). + For f16 source: takes 4 i32 values (packed f16x2). + For bf16 source: takes 4 i32 values (packed bf16x2). + + The `srcType` attribute specifies the source element type (f32, f16, + or bf16) and is used to select the correct PTX cvt instruction suffix. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + Variadic:$srcs, + OptionalAttr:$rnd, + TypeAttr:$srcType, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_DotAccumulate2WayOp : NVVM_Op<"dot.accumulate.2way"> { + let description = [{ + Performs a two-way 16-bit to 8-bit dot-product which is accumulated in a + 32-bit result. + Operand `a` is a vector of two 16-bit elements and operand `b` a vector + of four 8-bit elements between which the dot product is computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `s`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `u`, then the elements in the corresponding + vector are zero-extended to 32-bit instead. + + The `b_hi` boolean attribute specifies which two bytes of `b` are used for + the dot product. If `b_hi` is true, then the dot product is computed + between `a` and elements at indices 2 and 3 of `b`. If `b_hi` is false, + then the dot product is computed between `a` and elements at indices 0 and + 1 of `b`. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is + signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp2a) + }]; + let arguments = (ins + NVVM_VectorOf16BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type, + BoolAttr:$b_hi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_DotAccumulate4WayOp : NVVM_Op<"dot.accumulate.4way"> { + let description = [{ + Performs a four-way byte dot-product which is accumulated in a 32-bit + result. + Operand `a` and `b` are vectors of 4 bytes between which the dot product is + computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `signed`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `unsigned`, then the elements in the + corresponding vector are zero-extended to 32-bit instead. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is `s8`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp4a) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ElectSyncOp : NVVM_Op<"elect.sync"> { + let description = [{ + The `elect.sync` instruction elects one predicated active leader + thread from among a set of threads specified in the `membermask`. + When the `membermask` is not provided explicitly, a default value + of `0xFFFFFFFF` is used. The predicate result is set to `True` for + the leader thread, and `False` for all other threads. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync) + }]; + let arguments = (ins + Optional:$membermask + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$pred + ); +} +def NVVM_Ex2Op : NVVM_Op<"ex2", [Pure]> { + let description = [{ + Computes a fast approximation of 2 raised to the power of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_Exit : NVVM_Op<"exit"> { + let description = [{ + Ends execution of a thread. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#control-flow-instructions-exit) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FAbsOp : NVVM_InferredOp<"fabs"> { + let description = [{ + Returns the absolute value of the floating point operand. The Op also has an + optional `ftz` attribute which flushes subnormal inputs and returns to + sign-preserving zero + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#floating-point-instructions-abs) + }]; + let arguments = (ins + NVVM_FloatingPointOrFixedLengthVector:$value, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_FloatingPointOrFixedLengthVector:$res + ); +} +def NVVM_FenceMbarrierInitOp : NVVM_Op<"fence.mbarrier.init"> { + let description = [{ + Fence operation that applies on the prior nvvm.mbarrier.init + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceProxyOp : NVVM_Op<"fence.proxy"> { + let description = [{ + Fence operation with proxy to establish an ordering between memory accesses + that may happen through different proxies. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_ProxyKindAttr:$kind, + OptionalAttr:$space + ); + let results = (outs); +} +def NVVM_FenceProxyAcquireOp : NVVM_Op<"fence.proxy.acquire"> { + let description = [{ + `fence.proxy.acquire` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy + + The address operand `addr` and the operand `size` together specify the + memory range `[addr, addr+size)` on which the ordering guarantees on the + memory accesses across the proxies is to be provided. The only supported + value for the `size` operand is 128 and must be an immediate. Generic Addressing + is used unconditionally, and the address specified by the operand `addr` must + fall within the `.global` state space. Otherwise, the behavior is undefined + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$addr, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxyReleaseOp : NVVM_Op<"fence.proxy.release"> { + let description = [{ + `fence.proxy.release` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy. `fence.proxy.release` + operation can form a release sequence that synchronizes with an acquire + sequence that contains the fence.proxy.acquire proxy fence operation + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxySyncRestrictOp : NVVM_Op<"fence.proxy.sync_restrict"> { + let description = [{ + The `nvvm.fence.proxy.sync_restrict` Op used to establish + ordering between a prior memory access performed between proxies. Currently, + the ordering is only supported between async and generic proxies. `sync_restrict` + restricts `acquire` memory semantics to `shared_cluster` and `release` memory + semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceScClusterOp : NVVM_Op<"fence.sc.cluster"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceSyncRestrictOp : NVVM_Op<"fence.sync_restrict"> { + let description = [{ + The `nvvm.fence.sync_restrict` Op restricts the class of memory + operations for which the fence instruction provides the memory ordering guarantees. + `sync_restrict` restricts `acquire` memory semantics to `shared_cluster` and + `release` memory semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order + ); + let results = (outs); +} +def NVVM_FmaOp : NVVM_InferredOp<"fma"> { + let description = [{ + The `nvvm.fma` operation performs floating point fused multiply-add of + three operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, flush-to-zero by the `ftz` attribute, and ReLU by the + `relu` attribute. + + Out-of-bounds (OOB) behavior is controlled by the `oob` attribute. `oob` + clamps the result to 0 if either of the operands is `OOB NaN` (see [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)). + + For more information, see PTX ISA: + - [floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma) + - [half-precision floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-fma) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$a, + NVVM_T16BitFloatOrBfloat16Type:$b, + NVVM_T16BitFloatOrBfloat16Type:$c, + NVVM_FmaRndAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz, + OptionalAttr:$relu, + OptionalAttr:$oob + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_FMAPackedF32x2Op : NVVM_InferredOp<"fma.packed.f32x2"> { + let description = [{ + This Op performs FMA of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + NVVM_VectorOf32BitFloatValues2:$srcC, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_FMAPacked_F32x2_BF16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xbf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_FMAPacked_F32x2_F16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_MaxOp : NVVM_Op<"fmax"> { + let description = [{ + `nvvm.fmax` Op calculates maximum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the maximum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-max) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MinOp : NVVM_Op<"fmin"> { + let description = [{ + `nvvm.fmin` Op calculates minimum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the minimum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-min) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_GriddepcontrolOp : NVVM_Op<"griddepcontrol"> { + let description = [{ + If the $kind attribute is set to `wait`, it causes the + executing thread to wait until all prerequisite grids in flight + have completed and all the memory operations from the prerequisite grids + are performed and made visible to the current grid. + + When the $kind is launch_dependents, it signals that specific dependents + the runtime system designated to react to this instruction can be scheduled + as soon as all other CTAs in the grid issue the same instruction or have + completed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol) + }]; + let arguments = (ins + NVVM_GridDepActionAttr:$kind + ); + let results = (outs); +} +def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx", [AttrSizedOperandSegments]> { + let description = [{ + This op allows using PTX directly within the NVVM + dialect, while greatly simplifying llvm.inline_asm generation. It + automatically handles register size selection and sets the correct + read/write access for each operand. The operation leverages the + `BasicPtxBuilderInterface` to abstract away low-level details of + PTX assembly formatting. + + The `predicate` attribute is used to specify a predicate for the + PTX instruction. + + Example 1: Read-only Parameters + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count) : !llvm.ptr, i32 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "mbarrier.init.b64 [$0], $1;", "l,r" %arg0, %arg2 : (!llvm.ptr, i32) -> () + ``` + + Example 2: Read-only and Write-only Parameters + ```mlir + %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input) : f32 -> f32 + + // Lowers to: + %0 = llvm.inline_asm has_side_effects asm_dialect = att + "ex2.approx.ftz.f32 $0, $1;", "=f,f" %arg0 : (f32) -> f32 + ``` + + Example 3: Predicate Usage + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count), + predicate = %pred : !llvm.ptr, i32, i1 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b" %arg0, %arg2, %arg3 + : (!llvm.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + Variadic:$readOnlyArgs, + Variadic:$readWriteArgs, + Optional:$predicate, + StrAttr:$ptxCode + ); + let results = (outs + Optional:$writeOnlyArgs + ); +} +def NVVM_LdMatrixOp : NVVM_InferredOp<"ldmatrix"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + I32Attr:$num, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape, + OptionalAttr:$srcFormat + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_ExtLoadOp : NVVM_Op<"load.ext"> { + let description = [{ + The `load.ext` operation is used to read from memory in generic, + global or shared address space. This Op explicitly supports + various cache properties and memory orderings associated with + the load operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$prefetch, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace, + OptionalAttr:$unified + ); + let results = (outs + NVVM_LLVMTypeWithSize:$res + ); +} +def NVVM_Log2Op : NVVM_Op<"log2", [Pure]> { + let description = [{ + Computes a fast approximation of the base-2 logarithm of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MapaOp : NVVM_Op<"mapa"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$a, + NVVM_T32BitSignlessInteger:$b + ); + let results = (outs + NVVM_LLVMPointerInAddressSpace03:$res + ); +} +def NVVM_MatchSyncOp : NVVM_InferredOp<"match.sync"> { + let description = [{ + The `match.sync` op performs broadcast and compare of operand `val` across + all non-exited threads in `thread_mask` and returns a mask depending on the + kind and an optional predicate. + + The matching operation kinds are: + - `any`: Returns a mask corresponding to the non-exited threads in the + `thread_mask` that have the same value of operand `val`. + - `all`: Returns a mask and a predicate. If all non-exited threads in the + `thread_mask` have the same value of operand `val`, the predicate is set to + true and the mask corresponds to the non-exited threads in the + `thread_mask`. Otherwise, the predicate is set to false and the mask is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-match-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr64:$val, + NVVM_MatchSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierArriveOp : NVVM_InferredOp<"mbarrier.arrive", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive` operation performs an arrive-on operation on the + *mbarrier object* at the specified address. Uses the default `.release.cta` semantics. + This release pattern establishes memory ordering for operations occurring in program + order before this arrive instruction by making operations from the current thread + visible to subsequent operations in other threads within the CTA. When other threads + perform corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize + with this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier. + + - `res`: When the `space` is not shared_cluster, this operation returns an + opaque 64-bit value capturing the phase of the *mbarrier object* prior to + the arrive-on operation. The contents of this return value are + implementation-specific. An *mbarrier object* located in the shared_cluster + space cannot return a value. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared_cta or shared_cluster memory. When it + is generic, the underlying address must be within the shared_cta memory space; + otherwise the behavior is undefined. + - `count`: This specifies the amount by which the pending arrival count is + decremented. If the `count` argument is not specified, the pending arrival + count is decremented by 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.arrive` operation. + - `space`: This indicates the memory space where the mbarrier object resides. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveExpectTxOp : NVVM_InferredOp<"mbarrier.arrive.expect_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive.expect_tx` operation performs an expect-tx operation + followed by an arrive-on operation on the *mbarrier object*. Uses the default + `.release.cta` semantics. This release pattern establishes memory ordering for + operations occurring in program order before this arrive instruction by making + operations from the current thread visible to subsequent operations in other + threads within the CTA. When other threads perform corresponding acquire operations + (like 'mbarrier.test.wait'), they synchronize with this release pattern. + + This operation first performs an expect-tx operation with the specified transaction + count, then performs an arrive-on operation with an implicit count of 1. The + expect-tx operation increases the expect-count of the *mbarrier object* by the + specified value (i.e. `txcount`), setting the current phase to expect and track + the completion of additional asynchronous transactions. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `txcount`: An unsigned integer specifying the expected transaction count + for the expect-tx operation. This represents the number of asynchronous transactions + expected to complete before the barrier phase completes. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + - `predicate`: Optional predicate for conditional execution used only when lowering to + inline-ptx. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + Optional:$predicate, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveNocompleteOp : NVVM_Op<"mbarrier.arrive.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive.nocomplete` operation performs an arrive-on operation + on the *mbarrier object* with the guarantee that it will not cause the barrier to + complete its current phase. Uses the default `.release.cta` semantics. This release + pattern establishes memory ordering for operations occurring in program order before + this arrive instruction by making operations from the current thread visible to + subsequent operations in other threads within the CTA. When other threads perform + corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize with + this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier + with a specified count, but ensures that the barrier phase will not complete as + a result of this operation. The operation returns an opaque value that + captures the phase of the *mbarrier object* prior to the arrive-on operation. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the count argument to the arrive-on operation. + Must be in the valid range as specified in the *mbarrier object* contents. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierArriveDropOp : NVVM_InferredOp<"mbarrier.arrive_drop", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive_drop` operation decrements the expected arrival + count of the *mbarrier object* by `count` and then performs an arrive-on + operation. When `count` is not specified, it defaults to 1. The decrement + of the expected arrival count applies to all the subsequent phases of the + *mbarrier object*. The remaining semantics are identical to those of the + `nvvm.mbarrier.arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropExpectTxOp : NVVM_InferredOp<"mbarrier.arrive_drop.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.expect_tx` operation is similar to the + `nvvm.mbarrier.arrive.expect_tx` operation except that it performs an + `arrive_drop` operation instead of only an `arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropNocompleteOp : NVVM_Op<"mbarrier.arrive_drop.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.nocomplete` operation decrements the expected + arrival count of the *mbarrier object* by the amount `count` and then performs + an arrive-on operation on the *mbarrier object* with the guarantee that it + will not cause the barrier to complete its current phase. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCheckLayoutOp : NVVM_Op<"mbarrier.check.layout"> { + let description = [{ + The `nvvm.mbarrier.check.layout` operation returns `true` if the *mbarrier object* at `addr` + uses the given `layout`; otherwise it returns `false`. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$addr, + NVVM_MBarrierLayoutAttr:$layout + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCompleteTxOp : NVVM_Op<"mbarrier.complete_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.complete_tx` operation decrements the transaction + count of the *mbarrier object* at `addr` by `txcount`. It also signals + the completion of asynchronous transactions that were tracked by the + current phase. The `scope` specifies the set of threads that can directly + observe the memory synchronizing effect of the `mbarrier.complete_tx` + operation. `CTA` and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-complete-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$report, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierExpectTxOp : NVVM_Op<"mbarrier.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.expect_tx` operation increases the transaction count + of the mbarrier located at `addr` by `txcount` amount. The `scope` + specifies the set of threads that can directly observe the memory + synchronizing effect of the `mbarrier.expect_tx` operation. `CTA` + and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-expect-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierInitOp : NVVM_Op<"mbarrier.init"> { + let description = [{ + The `nvvm.mbarrier.init` operation initializes an *mbarrier object* at the specified + memory location. + + This operation initializes the *mbarrier object* with the following state: + - Current phase: 0 + - Expected arrival count: `count` + - Pending arrival count: `count` + - Transaction count (tx-count): 0 + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the number of threads that will participate in barrier + synchronization. Must be in the range [1, 2²⁰ - 1]. + - `predicate`: Optional predicate for conditional execution. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$predicate, + OptionalAttr:$layout + ); + let results = (outs); +} +def NVVM_MBarrierInvalOp : NVVM_Op<"mbarrier.inval"> { + let description = [{ + The `nvvm.mbarrier.inval` operation invalidates an *mbarrier object* at the + specified memory location. + + This operation marks the *mbarrier object* as invalid, making it safe to repurpose + the memory location for other uses or to reinitialize it as a new *mbarrier object*. + It is undefined behavior if the *mbarrier object* is already invalid. + + The operation takes the following operand: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-inval) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr + ); + let results = (outs); +} +def NVVM_MBarrierTestWaitOp : NVVM_InferredOp<"mbarrier.test.wait"> { + let description = [{ + The `nvvm.mbarrier.test.wait` operation performs a non-blocking test for the + completion of a specific phase of an *mbarrier object*. It uses the default + `.acquire.cta` semantics. This acquire pattern establishes memory ordering for + operations occurring in program order after this wait instruction by making + operations from other threads in the CTA visible to subsequent operations in the current + thread. When this wait completes, it synchronizes with the corresponding release + pattern from the `mbarrier.arrive` operation, establishing memory ordering within + the CTA. + + This operation tests whether the mbarrier phase specified by the state operand + has completed. It is a non-blocking instruction that immediately returns the + completion status without suspending the executing thread. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `stateOrPhase`: This argument represents a `state` when it is a 64-bit value + and represents a `phase` when it is a 32-bit value. The `state` is an opaque + value returned by a previous `mbarrier.arrive` operation on the same + *mbarrier object* during the current or immediately preceding phase. + The `phase` is an integer specifying the phase parity (0 or 1). + Even phases have parity 0, odd phases have parity 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + The operation returns a boolean value indicating whether the specified phase + has completed: + - `true`: The immediately preceding phase has completed + - `false`: The phase is still incomplete (current phase) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitOp : NVVM_InferredOp<"mbarrier.try_wait"> { + let description = [{ + The `nvvm.mbarrier.try_wait` operation checks whether the specified + *mbarrier object* at `addr` has completed the given phase. Note that + unlike the `nvvm.mbarrier.test.wait` operation, the try_wait operation + is a potentially-blocking one. If the phase is not yet complete, the + calling thread may be suspended. A suspended thread resumes execution + once the phase completes or when a system-defined timeout occurs. + Optionally, the `ticks` operand can be used to provide a custom timeout + (in nanoseconds), overriding the system-defined one. The semantics of + this operation and its operands are otherwise similar to those of the + `nvvm.mbarrier.test.wait` Op. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + Optional:$ticks, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity"> { + let description = [{ + The `nvvm.mbarrier.try_wait.parity` operation performs a potentially-blocking + test for the completion of a specific phase of an *mbarrier object* using phase + parity. It uses the default `.acquire.cta` semantics. This acquire pattern + establishes memory ordering for operations occurring in program order after this + wait instruction by making operations from other threads in the CTA visible to subsequent + operations in the current thread. When this wait completes, it synchronizes with + the corresponding release pattern from the `mbarrier.arrive` operation, establishing + memory ordering within the CTA. + + This operation waits for the completion of the mbarrier phase indicated by the + phase parity. While it uses the underlying PTX `mbarrier.try_wait.parity` + instruction, this MLIR operation generates a loop that enforces the test to + complete before continuing execution, ensuring the barrier phase is actually + completed rather than potentially timing out. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `phase`: An integer specifying the phase parity (0 or 1). Even phases + have parity 0, odd phases have parity 1. + - `ticks`: An unsigned integer specifying the suspend time hint in + nanoseconds. This may be used instead of the system-dependent time limit. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + **Implementation behavior**: + This operation generates a PTX loop that repeatedly calls the underlying + `mbarrier.try_wait.parity` instruction until the barrier phase completes. + Unlike the raw PTX instruction which may return without completion after a + timeout, this MLIR operation guarantees completion by continuing to loop until + the specified phase is reached. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$ticks, + OptionalAttr:$useIntrinsic + ); + let results = (outs); +} +def NVVM_MBarrierTryWaitParityTimeLimitOp : NVVM_Op<"mbarrier.try_wait.parity.timelimit"> { + let description = [{ + The `mbarrier.try_wait.parity.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `phase` operand + is the integer parity of either the current phase or the immediately + preceding phase of the mbarrier object. The `order` denotes the + ordering semantics for the memory consistency model. This Op only + supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTryWaitTimeLimitOp : NVVM_Op<"mbarrier.try_wait.timelimit"> { + let description = [{ + The `mbarrier.try_wait.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `state` operand + is the captured state of the mbarrier object from prior operations. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTransactionOp : NVVM_Op<"mbarrier.txn"> { + let description = [{ + The `mbarrier.txn` Op exposes the transactions that can be done on an MBarrier object. + + Operands: + - `addr`: Address of the mbarrier in generic or shared memory space + - `count`: Transaction count for the given kind + + - `multicast`: (Optional) Multicast mask for cluster space operations + + + Attributes: + - `kind`: One of the transaction kinds from MBarrierTxnKind + - `space`: Memory space (CTA or CLUSTER) where MBarrier object is located + - `scope`: Scope of synchronization (CTA or CLUSTER) + - `order`: Memory ordering semantics (release or relaxed only) + + This Op supports MBarrier objects in both shared::CTA and shared::CLUSTER spaces. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$multicast, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$space, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs); +} +def NVVM_MBarrierTransactionCTASpaceOp : NVVM_Op<"mbarrier.txn.cta"> { + let description = [{ + The `mbarrier.txn.cta` Op exposes the transactions that can be done + using an MBarrier object. The `addr` operand specifies the address of + the mbarrier in generic or shared memory space. The `kind` is one of + the various transaction kinds supported in `MBarrierTxnKind`. The + `count` argument is the transaction count for a given `kind`. The + `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'release' or 'relaxed' ordering. This + Op supports only MBarrier objects located in shared::CTA space (but + not in shared::CLUSTER space) and hence the 'CTASpace' suffix in the + naming. This Op returns an opaque 64-bit value capturing the phase of + the mbarrier object prior to the transaction. + [For more information on 'order/space/scope/kind-of-transactions' refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitOp : NVVM_Op<"mbarrier.wait"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `state` operand is the captured state of the mbarrier object + from prior operations. The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitParityOp : NVVM_Op<"mbarrier.wait.parity"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `phase` operand is the integer parity of either the current + phase or the immediately preceding phase of the mbarrier object. + The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MembarOp : NVVM_Op<"memory.barrier"> { + let description = [{ + `membar` operation guarantees that prior memory accesses requested by this + thread are performed at the specified `scope`, before later memory + operations requested by this thread following the membar instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_MmaBlockScaleOp : NVVM_Op<"mma.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.block_scale` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + All the threads in the warp must execute the same `mma.block_scale` operation. + + This operation follows the same design pattern as `nvvm.mma.sync`, with additional + scaling operands for both A and B matrices. + + Example: + ```mlir + %d = nvvm.mma.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<4xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaBlockScaleOp : NVVM_InferredOp<"mma.block_scale.internal"> { + let description = [{ + The `nvvm.mma.blockscale.internal` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factor. + Dimensions of SF_A and SF_B are based on scale vector sizes (X1, X2, X4), + the data type of must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %0 = nvvm.mma.block_scale.internal %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format + kind = #nvvm.block_scale_kind + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_MMAShapeAttr:$shape, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + OptionalAttr:$kind + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpBlockScaleOp : NVVM_Op<"mma.sp.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.block_scale` operation collectively performs the operation + `D = matmul(A_sparse * SF_A, B * SF_B) + C` using all threads in a warp. + + A is a sparse matrix, and B, C and D are dense matrices. + SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + This operation is similar to `nvvm.mma.block_scale` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.block_scale` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + This operation follows the same design pattern as `nvvm.mma.sp.sync`, with additional + scaling operands for both A and B matrices. Note that sparse block scale operations + always use ordered metadata (sm_90+). + + Example: + ```mlir + %d = nvvm.mma.sp.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<2xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind, + UnitAttr:$orderedMetadata + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaSparseBlockScaleOp : NVVM_InferredOp<"mma.sp.block_scale.internal"> { + let description = [{ + The `nvvm.mma.sparse.blockscale` operation collectively performs the + operation `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + Where A is structued sparse with atleast 50% zeros and B, C and D are + dense and SF_A and SF_B are scaling factor. Dimensions of SF_A and SF_B + are based on scale vector sizes (X1, X2, X4), the data type of + must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %selector = llvm.mlir.constant(0) + %0 = nvvm.mma.sparse.block_scale + %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC, %spMetadata, %selector + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + sparityFormat = #nvvm.sparsity_format + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_T32BitSignlessInteger:$sp_metadata, + NVVM_T32BitSignlessInteger:$selector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_SparsityFormatAttr:$sparsityFormat + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpOp : NVVM_Op<"mma.sp.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.sync` operation collectively performs the sparse operation + `D = matmul(A_sparse, B) + C` using all threads in a warp. + + This operation is similar to `nvvm.mma.sync` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.sync` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + The optional `orderedMetadata` attribute specifies the metadata ordering: + - Absence (default): Uses standard sparse metadata ordering + - Presence: Uses ordered metadata (PTX ISA 8.5+, sm_90+) + + The optional `kind` attribute specifies mixed-precision modes for FP8 operations: + - `f8f6f4`: Enables e3m2, e2m3, e2m1 FP8 types and f16 accumulator (PTX ISA 8.7+, sm_90+) + - Only valid with ordered metadata and m16n8k64 shape + + The shapes, layouts, and data types follow the same constraints as the + regular `nvvm.mma.sync` operation, but the A operand contains only the + non-zero elements in compressed format. + + Example: + ```mlir + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + + // With ordered metadata: + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {orderedMetadata, shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$intOverflowBehavior, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + UnitAttr:$orderedMetadata, + OptionalAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sync` operation collectively performs the operation + `D = matmul(A, B) + C` using all threads in a warp. + + All the threads in the warp must execute the same `mma.sync` operation. + + For each possible multiplicand PTX data type, there are one or more possible + instruction shapes given as "mMnNkK". The below table describes the posssibilities + as well as the types required for the operands. Note that the data type for + C (the accumulator) and D (the result) can vary independently when there are + multiple possibilities in the "C/D Type" column. + + When an optional attribute cannot be immediately inferred from the types of + the operands and the result during parsing or validation, an error will be + raised. + + `b1Op` is only relevant when the binary (b1) type is given to + `multiplicandDataType`. It specifies how the multiply-and-acumulate is + performed and is either `xor_popc` or `and_poc`. The default is `xor_popc`. + + `intOverflowBehavior` is only relevant when the `multiplicandType` attribute + is one of `u8, s8, u4, s4`, this attribute describes how overflow is handled + in the accumulator. When the attribute is `satfinite`, the accumulator values + are clamped in the int32 range on overflow. This is the default behavior. + Alternatively, accumulator behavior `wrapped` can also be specified, in + which case overflow wraps from one end of the range to the other. + + `layoutA` and `layoutB` are required and should generally be set to + `#nvvm.mma_layout` and `#nvvm.mma_layout` respectively, but other + combinations are possible for certain layouts according to the table below. + + ``` + | A/B Type | Shape | ALayout | BLayout | A Type | B Type | C/D Type | + |----------|-----------|---------|---------|----------|----------|-------------------| + | f64 | .m8n8k4 | row | col | 1x f64 | 1x f64 | 2x f64 | + | f16 | .m8n8k4 | row/col | row/col | 2x f16x2 | 2x f16x2 | 4x f16x2 or 8xf32 | + | | .m16n8k8 | row | col | 2x f16x2 | 1x f16x2 | 2x f16x2 or 4 f32 | + | | .m16n8k16 | row | col | 4x f16x2 | 2x f16x2 | 2x f16x2 or 4 f32 | + | bf16 | .m16n8k8 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k16 | row | col | 4x i32 | 2x i32 | 4x f32 | + | tf32 | .m16n8k4 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k8 | row | col | 4x i32 | 2x i32 | 2x f16x2 or 4 f32 | + | u8/s8 | .m8n8k16 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | .m16n8k16 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | .m16n8k32 | row | col | 4x i32 | 2x i32 | 4x i32 | + | u4/s4 | .m8n8k32 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k32 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | m16n8k64 | row | col | 4x i32 | 2x i32 | 4x i32 | + | b1 | m8n8k128 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k128 | row | col | 2x i32 | 1x i32 | 4x i32 | + ``` + + Example: + ```mlir + + %128 = nvvm.mma.sync A[%120, %121, %122, %123] + B[%124, %125] + C[%126, %127] + {layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = {k = 16 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) + -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaSmemDescOp : NVVM_Op<"mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.mma_smem_desc` constructs a Shared Memory descriptor for use as the MMA Shared Memory operand. + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$pointer, + NVVM_T32BitSignlessInteger:$ldm, + NVVM_T32BitSignlessInteger:$stride, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T8BitSignlessInteger2:$swizzle, + OptionalAttr:$mmaDescVersion + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MovMatrixOp : NVVM_Op<"movmatrix"> { + let description = [{ + Moves a row-major matrix across all threads in a warp, reading elements + from source `$src`, and writing the transposed elements to destination + `$dst`. + + The `shape` attribute indicates the dimensions of the matrix being + transposed. Each matrix element holds 16-bit data as indicated by the + `eltType` attribute. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-movmatrix-instruction) + + Example: + ```mlir + %dst = nvvm.movmatrix %src {shape = #nvvm.ld_st_matrix_shape, + eltType = #nvvm.ld_st_matrix_elt_type} : i32 + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$src, + NVVM_LdStMatrixShapeAttr:$shape, + OptionalAttr:$layout, + NVVM_LdStMatrixEltTypeAttr:$eltType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_MulOp : NVVM_InferredOp<"mul"> { + let description = [{ + nvvm.mul Op multiplies two values with a given `type` and `mode`. + The type of the operation represents the types of the `a` and `b` operands. + If `.hi` or `.lo` is specified, then `res` is the same size as `a` and `b`, + and either the upper or lower half of the result is written to the + destination register. If `.wide` is specified, then `res` is twice as + wide as `a` and `b` to receive the full result of the multiplication. The + `.wide` suffix is supported only for 16- and 32-bit integer types. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-mul) + }]; + let arguments = (ins + NVVM_T16BitSignlessIntegerOr323:$a, + NVVM_T16BitSignlessIntegerOr323:$b, + NVVM_MulModeAttr:$mode, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr323:$res + ); +} +def NVVM_MulPacked_BF16x2_BF16x2_F16x2Op : NVVM_Op<"mul.packed.bf16x2.bf16x2.f16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "bf16"x2 and "f16"x2 vectors. + The inputs are of the form v2bf16 and v2f16, and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.bf16x2.bf16x2.f16x2 %srcA, %srcB : vector<2xbf16>, vector<2xf16> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf16BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_F16x2_F16x2_BF16x2Op : NVVM_Op<"mul.packed.f16x2.f16x2.bf16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "f16"x2 and "bf16"x2 vectors. + The inputs are of the form v2f16 and v2bf16, and the output is of the form v2f16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.f16x2.f16x2.bf16x2 %srcA, %srcB : vector<2xf16>, vector<2xbf16> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOfBfloat16TypeValuesOf:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2f16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPackedF32x2Op : NVVM_InferredOp<"mul.packed.f32x2"> { + let description = [{ + This Op multiplies floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_MultimemLdReduceOp : NVVM_Op<"multimem.ld.reduce"> { + let description = [{ + The `nvvm.multimem.ld.reduce` operation performs a multimem load-reduce + instruction. It reads from a multimem address and reduces partial + results across participating threads. + + The `ordering` attribute specifies the memory ordering semantics (weak, + relaxed, acquire). + + The `op` attribute specifies the reduction operation (and, or, + xor, add, min, max). + + The `dataType` attribute specifies the PTX data type: + - Integer: b32, s32, u32, b64, s64, u64 + - Floating-point: f16, bf16, f32, f64 + - FP8: e5m2, e4m3 + + The optional `accPrec` attribute specifies the accumulator precision type + for sub-single-precision types (f16/bf16/f8E5M2/f8E4M3FN): f16 or f32. + + Example: + ```mlir + %res = nvvm.multimem.ld.reduce %addr + {ordering = #nvvm.mem_order, + op = #nvvm.red_op, + dataType = f32} + : !llvm.ptr<1> -> f32 + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemLdReduceOrderingAttr:$ordering, + NVVM_MultimemLdReduceOpAttr:$op, + TypeAttr:$dataType, + OptionalAttr:$accPrec + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_MultimemRedOp : NVVM_Op<"multimem.red"> { + let description = [{ + The `nvvm.multimem.red` operation performs a multimem atomic reduction, + applying a reduction operation between a source value and the value at a + multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `op` attribute specifies the reduction operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point types). + + Example: + ```mlir + nvvm.multimem.red %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + op = #nvvm.red_op, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_AnyType:$addr, + NVVM_MultimemRedOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + NVVM_MultimemRedOpAttr:$op, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_MultimemStOp : NVVM_Op<"multimem.st"> { + let description = [{ + The `nvvm.multimem.st` operation performs a multimem store instruction, + broadcasting a value to a multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point, e5m2/e4m3 for + FP8 types). + + Example: + ```mlir + nvvm.multimem.st %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemStOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_NanosleepOp : NVVM_Op<"nanosleep"> { + let description = [{ + The op suspends the thread for a sleep duration approximately close to the + delay `$duration`, specified in nanoseconds. + + The sleep duration is approximated, but guaranteed to be in the + interval [0, 2*t]. The maximum sleep duration is 1 millisecond. + The implementation may reduce the sleep duration for individual threads + within a warp such that all sleeping threads in the warp wake up together. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-nanosleep) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$duration + ); + let results = (outs); +} +def NVVM_PMEventOp : NVVM_Op<"pmevent"> { + let description = [{ + Triggers one or more of a fixed number of performance monitor events, with + event index or mask specified by immediate operand. + + Without `mask` it triggers a single performance monitor event indexed by + immediate operand a, in the range 0..15. + + With `mask` it triggers one or more of the performance monitor events. Each + bit in the 16-bit immediate operand controls an event. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent) + }]; + let arguments = (ins + OptionalAttr:$maskedEventId, + OptionalAttr:$eventId + ); + let results = (outs); +} +def NVVM_PrefetchOp : NVVM_Op<"prefetch"> { + let description = [{ + Prefetches the cache line containing the address given by `addr`. The + operand may be a global, local, or generic pointer. When `tensormap` is + specified, the operand may instead be a constant or generic pointer. If the + address maps to shared memory, the operation has no effect. + + At most one of `cacheLevel` or `tensormap` may be present. The `cacheLevel` + attribute selects the target cache level. When combined with `uniform`, the + prefetch is performed to the uniform cache, in which case `addr` must be a + generic pointer. + + When `tensormap` is used, the line containing `addr` is brought from the + constant or parameter state space for later use by `cp.async.bulk.tensor`. + If `in_param_space` is specified, the generic pointer is interpreted as + referring to the parameter state space. + + `uniform` can be specified after the `cacheLevel` to indicate that the + prefetch is performed to the specified uniform cache level. If `uniform` is + specified, `addr` must be a generic address pointer and no operation is + performed if `addr` maps to a `const`, `local`, or `shared` memory location. + + The `evictPriority` attribute is optional and specifies the cache eviction + priority when `cacheLevel` is L2. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace13:$addr, + Optional:$predicate, + OptionalAttr:$cacheLevel, + OptionalAttr:$evictPriority, + UnitAttr:$tensormap, + UnitAttr:$uniform, + UnitAttr:$in_param_space + ); + let results = (outs); +} +def NVVM_PermuteOp : NVVM_Op<"prmt", [Pure]> { + let description = [{ + The `nvvm.prmt` operation constructs a permutation of the + bytes of the first one or two operands, selecting based on + the 2 least significant bits of the final operand. + + The bytes in the first one or two source operands are numbered. + The first source operand (%lo) is numbered {b3, b2, b1, b0}, + in the case of the '``default``', '``f4e``' and '``b4e``' variants, + the second source operand (%hi) is numbered {b7, b6, b5, b4}. + + Modes: + - `default`: Index mode - each nibble in `selector` selects a byte from the 8-byte pool + - `f4e` : Forward 4 extract - extracts 4 contiguous bytes starting from position in `selector` + - `b4e` : Backward 4 extract - extracts 4 contiguous bytes in reverse order + - `rc8` : Replicate 8 - replicates the lower 8 bits across the 32-bit result + - `ecl` : Edge clamp left - clamps out-of-range indices to the leftmost valid byte + - `ecr` : Edge clamp right - clamps out-of-range indices to the rightmost valid byte + - `rc16` : Replicate 16 - replicates the lower 16 bits across the 32-bit result + + Depending on the 2 least significant bits of the %selector operand, the result + of the permutation is defined as follows: + + +------------+----------------+--------------+ + | Mode | %selector[1:0] | Output | + +------------+----------------+--------------+ + | '``f4e``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {4, 3, 2, 1} | + | +----------------+--------------+ + | | 2 | {5, 4, 3, 2} | + | +----------------+--------------+ + | | 3 | {6, 5, 4, 3} | + +------------+----------------+--------------+ + | '``b4e``' | 0 | {5, 6, 7, 0} | + | +----------------+--------------+ + | | 1 | {6, 7, 0, 1} | + | +----------------+--------------+ + | | 2 | {7, 0, 1, 2} | + | +----------------+--------------+ + | | 3 | {0, 1, 2, 3} | + +------------+----------------+--------------+ + | '``rc8``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 1} | + | +----------------+--------------+ + | | 2 | {2, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecl``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 1, 1} | + | +----------------+--------------+ + | | 2 | {3, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecr``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 0} | + | +----------------+--------------+ + | | 2 | {2, 2, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 1, 0} | + +------------+----------------+--------------+ + | '``rc16``' | 0 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 3, 2} | + | +----------------+--------------+ + | | 2 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 3, 2} | + +------------+----------------+--------------+ + + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prmt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$lo, + Optional:$hi, + NVVM_T32BitSignlessInteger:$selector, + NVVM_PermuteModeAttr:$mode + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_RcpApproxFtzF32Op : NVVM_Op<"rcp.approx.ftz.f", [Pure]> { + let arguments = (ins + NVVM_T32BitFloat:$arg + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_AggrSmemSize : NVVM_Op<"read.ptx.sreg.aggr.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClockOp : NVVM_Op<"read.ptx.sreg.clock"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_Clock64Op : NVVM_Op<"read.ptx.sreg.clock64"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdXOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdYOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdZOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterId : NVVM_Op<"read.ptx.sreg.cluster.ctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksXOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksYOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksZOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDim : NVVM_Op<"read.ptx.sreg.cluster.nctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdXOp : NVVM_Op<"read.ptx.sreg.clusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdYOp : NVVM_Op<"read.ptx.sreg.clusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdZOp : NVVM_Op<"read.ptx.sreg.clusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdXOp : NVVM_Op<"read.ptx.sreg.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdYOp : NVVM_Op<"read.ptx.sreg.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdZOp : NVVM_Op<"read.ptx.sreg.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_DynamicSmemSize : NVVM_Op<"read.ptx.sreg.dynamic.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg0Op : NVVM_Op<"read.ptx.sreg.envreg0", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg1Op : NVVM_Op<"read.ptx.sreg.envreg1", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg10Op : NVVM_Op<"read.ptx.sreg.envreg10", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg11Op : NVVM_Op<"read.ptx.sreg.envreg11", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg12Op : NVVM_Op<"read.ptx.sreg.envreg12", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg13Op : NVVM_Op<"read.ptx.sreg.envreg13", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg14Op : NVVM_Op<"read.ptx.sreg.envreg14", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg15Op : NVVM_Op<"read.ptx.sreg.envreg15", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg16Op : NVVM_Op<"read.ptx.sreg.envreg16", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg17Op : NVVM_Op<"read.ptx.sreg.envreg17", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg18Op : NVVM_Op<"read.ptx.sreg.envreg18", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg19Op : NVVM_Op<"read.ptx.sreg.envreg19", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg2Op : NVVM_Op<"read.ptx.sreg.envreg2", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg20Op : NVVM_Op<"read.ptx.sreg.envreg20", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg21Op : NVVM_Op<"read.ptx.sreg.envreg21", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg22Op : NVVM_Op<"read.ptx.sreg.envreg22", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg23Op : NVVM_Op<"read.ptx.sreg.envreg23", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg24Op : NVVM_Op<"read.ptx.sreg.envreg24", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg25Op : NVVM_Op<"read.ptx.sreg.envreg25", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg26Op : NVVM_Op<"read.ptx.sreg.envreg26", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg27Op : NVVM_Op<"read.ptx.sreg.envreg27", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg28Op : NVVM_Op<"read.ptx.sreg.envreg28", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg29Op : NVVM_Op<"read.ptx.sreg.envreg29", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg3Op : NVVM_Op<"read.ptx.sreg.envreg3", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg30Op : NVVM_Op<"read.ptx.sreg.envreg30", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg31Op : NVVM_Op<"read.ptx.sreg.envreg31", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg4Op : NVVM_Op<"read.ptx.sreg.envreg4", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg5Op : NVVM_Op<"read.ptx.sreg.envreg5", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg6Op : NVVM_Op<"read.ptx.sreg.envreg6", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg7Op : NVVM_Op<"read.ptx.sreg.envreg7", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg8Op : NVVM_Op<"read.ptx.sreg.envreg8", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg9Op : NVVM_Op<"read.ptx.sreg.envreg9", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerOp : NVVM_Op<"read.ptx.sreg.globaltimer"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerLoOp : NVVM_Op<"read.ptx.sreg.globaltimer.lo"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridIdOp : NVVM_Op<"read.ptx.sreg.gridid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneIdOp : NVVM_Op<"read.ptx.sreg.laneid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskEqOp : NVVM_Op<"read.ptx.sreg.lanemask.eq", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGeOp : NVVM_Op<"read.ptx.sreg.lanemask.ge", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGtOp : NVVM_Op<"read.ptx.sreg.lanemask.gt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLeOp : NVVM_Op<"read.ptx.sreg.lanemask.le", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLtOp : NVVM_Op<"read.ptx.sreg.lanemask.lt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimXOp : NVVM_Op<"read.ptx.sreg.nclusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimYOp : NVVM_Op<"read.ptx.sreg.nclusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimZOp : NVVM_Op<"read.ptx.sreg.nclusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimXOp : NVVM_Op<"read.ptx.sreg.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimYOp : NVVM_Op<"read.ptx.sreg.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimZOp : NVVM_Op<"read.ptx.sreg.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmDimOp : NVVM_Op<"read.ptx.sreg.nsmid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimXOp : NVVM_Op<"read.ptx.sreg.ntid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimYOp : NVVM_Op<"read.ptx.sreg.ntid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimZOp : NVVM_Op<"read.ptx.sreg.ntid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpDimOp : NVVM_Op<"read.ptx.sreg.nwarpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmIdOp : NVVM_Op<"read.ptx.sreg.smid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdXOp : NVVM_Op<"read.ptx.sreg.tid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdYOp : NVVM_Op<"read.ptx.sreg.tid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdZOp : NVVM_Op<"read.ptx.sreg.tid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_TotalSmemSize : NVVM_Op<"read.ptx.sreg.total.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpIdOp : NVVM_Op<"read.ptx.sreg.warpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpSizeOp : NVVM_Op<"read.ptx.sreg.warpsize", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_RedOp : NVVM_Op<"red"> { + let description = [{ + Performs a reduction operation with operand `b` and the value in + location `a`, and stores the result of the specified operation at + location `a`. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses + using Generic Addressing + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-red) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace05:$a, + NVVM_T32BitSignlessIntegerOr642:$b, + Optional:$cacheHint, + OptionalAttr:$memOrder, + OptionalAttr:$memScope, + NVVM_ReductionOpAttr:$op, + NVVM_ReductionTypeAttr:$type + ); + let results = (outs); +} +def NVVM_ReduxOp : NVVM_InferredOp<"redux.sync"> { + let description = [{ + `redux.sync` performs a reduction operation `kind` of the 32 bit source + register across all non-exited threads in the membermask. + + The `abs` and `nan` attributes can be used in the case of f32 input type, + where the `abs` attribute causes the absolute value of the input to be used + in the reduction operation, and the `nan` attribute causes the reduction + operation to return NaN if any of the inputs to participating threads are + NaN. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ReductionKindAttr:$kind, + OptionalAttr:$abs, + OptionalAttr:$nan + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr32:$res + ); +} +def NVVM_RsqrtOp : NVVM_InferredOp<"rsqrt", [Pure]> { + let description = [{ + Computes an approximation of the reciprocal of the square root of the + input value: `d = 1 / sqrt(a)`. Supports both f32 and f64. The maximum + relative error for the f32 form over the entire positive finite range + is 2^-22.9. + + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. For f64 inputs, `ftz=true` selects a coarser + approximation that uses only the upper 32 bits of the input (the lower + 32 bits of the result are zeroed). + + For more information, see PTX ISA: + [rsqrt](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-rsqrt) + }]; + let arguments = (ins + NVVM_T32BitFloatOr64Bit:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloatOr64Bit:$res + ); +} +def NVVM_SetMaxRegisterOp : NVVM_Op<"setmaxregister"> { + let arguments = (ins + I32Attr:$regCount, + NVVM_SetMaxRegisterActionAttr:$action + ); + let results = (outs); +} +def NVVM_ShflOp : NVVM_InferredOp<"shfl.sync"> { + let description = [{ + The `shfl.sync` Op implements data shuffle within threads of a warp. + The `thread_mask` denotes the threads participating in the Op where + the bit position corresponds to a particular thread's laneid. + The `offset` specifies a source lane or source lane offset + (depending on `kind`). The `val` is the input value to be copied from + the source. The `mask_and_clamp` contains two packed values specifying + a mask for logically splitting warps into sub-segments and an upper bound + for clamping the source lane index. + + The `return_value_and_is_valid` unit attribute can be specified to indicate + that the return value is a two-element struct, where the first element is + the result value and the second element is a predicate indicating if the + computed source lane index is valid. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$offset, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ShflKindAttr:$kind, + UnitAttr:$return_value_and_is_valid + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr322:$res + ); +} +def NVVM_SinOp : NVVM_Op<"sin", [Pure]> { + let description = [{ + Computes a fast approximation of the sine of the input value (in radians). + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. + + For more information, see PTX ISA: + [sin](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sin) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_SPCompressOp : NVVM_Op<"spcompress"> { + let description = [{ + Compresses elements from a dense vector `data` into a structured sparse vector `compressed_data`. + The metadata information is stored in `metadata`. + + Arguments: + - `spDesc`: Sparsity descriptor specifying the selection criteria for elements + - `data`: Input dense vector containing elements to be compressed + - `compFactor`: Compression factor attribute (only `sp2to4` is allowed, resulting in 2:4 compression) + - `indexSize`: Size of metadata index for each selected element (can only be `b2` or `b4` given 2:4 compression) + - `elemSize`: Size in bits for each element from input vector `data` + - `repFactor`: Repeat factor for the operation + + Results: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Output compressed vector containing selected elements + + Examples: + // Example 1: Basic compression with 4-bit elements, 2-bit indices, repeat factor of 1 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> -> i32, i32 + ``` + + // Example 2: Compression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> -> i32, vector<2xi32> + ``` + + // Example 3: Compression with 16-bit elements, 4-bit indices, larger repeat factor + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$spDesc, + NVVM_VectorOf32BitSignlessInteger:$data, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressIndexSizeAttr:$indexSize, + NVVM_SPCompressElemSizeAttr:$elemSize, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data + ); +} +def NVVM_SPDecompressOp : NVVM_Op<"spdecompress"> { + let description = [{ + Decompresses elements from a structured sparse vector `compressed_data` into a dense vector `data`. + The metadata information is stored in `metadata`. + + Arguments: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Compressed vector containing selected elements + - `compFactor`: Decompression factor attribute (sp1to2, sp1to4, sp1to8, sp1to16, sp2to4, sp2to8, sp2to16, sp4to8, sp4to16, sp8to16) + - `indexSize`: Size of metadata index for each selected element (b2 or b4) + - `elemSize`: Size in bits for each element from output vector `data` (b4, b8, b16) + - `repFactor`: Repeat factor for the operation (x1, x2, x4, x8, x16, x32, x64) + + Results: + - `data`: Output dense vector containing decompressed elements + + Examples: + // Example 1: Decompression with 4-bit elements, 2-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> i32 + ``` + + // Example 2: Decompression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> vector<2 x i32> + ``` + + // Example 3: Decompression with 16-bit elements, 4-bit indices, repeat factor of 64 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : vector<16 x i32>, vector<64 x i32> -> vector<128 x i32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data, + NVVM_SPDecompressFactorTypeAttr:$compFactor, + NVVM_SPDecompressIndexSizeAttr:$indexSize, + NVVM_SPDecompressElemSizeAttr:$elemSize, + NVVM_SPDecompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger2:$data + ); +} +def NVVM_BulkStoreOp : NVVM_Op<"st.bulk"> { + let description = [{ + Initializes a region of shared memory at the address given by `addr`. + The `size` operand specifies the number of bytes to initialize and must be + a multiple of 8. + The `initVal` operand specifies the value to initialize the memory to. The + only supported value is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessInteger:$size, + OptionalAttr:$initVal + ); + let results = (outs); +} +def NVVM_StMatrixOp : NVVM_Op<"stmatrix"> { + let description = [{ + Collectively store one or more matrices across all threads in a warp to the + location indicated by the address operand $ptr in shared memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$ptr, + Variadic:$sources, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape + ); + let results = (outs); +} +def NVVM_AsyncStoreOp : NVVM_Op<"store.async"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$addr, + NVVM_T32BitSignlessIntegerOr643:$value, + NVVM_LLVMPointerInAddressSpace7:$mbarrier + ); + let results = (outs); +} +def NVVM_ExtStoreOp : NVVM_Op<"store.ext"> { + let description = [{ + The `store.ext` operation is used to write to memory in generic, + global or shared address space. This Op explicitly supports various + cache properties and memory orderings associated with the store operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st) + }]; + let arguments = (ins + NVVM_LLVMTypeWithSize:$value, + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace + ); + let results = (outs); +} +def NVVM_SubPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_SubPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_SubPackedF32x2Op : NVVM_InferredOp<"sub.packed.f32x2"> { + let description = [{ + This Op performs subtraction of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_SubPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubFOp : NVVM_InferredOp<"subf", [Pure]> { + let description = [{ + The `nvvm.subf` operation performs floating point subtraction of two + operands. + + It supports the same type combinations and modifiers as `nvvm.addf`. + This is equivalent to `nvvm.addf(lhs, -rhs)`. + + For more information, see PTX ISA: + - [floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sub) + - [half-precision floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-sub) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_Tcgen05AllocOp : NVVM_Op<"tcgen05.alloc"> { + let description = [{ + The `tcgen05.alloc` Op allocates tensor core memory for + the amount specified by `nCols` and writes the destination + address to the `addr` argument. The `nCols` operand specifies the + number of columns to be allocated and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CommitOp : NVVM_Op<"tcgen05.commit"> { + let description = [{ + The `tcgen05.commit` makes the *mbarrier object*, specified by + the operand `addr`, track the completion of all the prior + async-tcgen05 operations initiated by the executing thread. + The multicast variants allow signaling on the *mbarrier objects* + of multiple CTAs within the cluster. Operand `multicastMask`, + when present, specifies the destination CTAs in the cluster such + that each bit position in the 16-bit or 32-bit `multicastMask` operand + corresponds to the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + Operand `smem_a_read` when present, specifies track only shared memory + read of "Matrix A for the prior tcgen05.mma Ops" + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + Optional:$multicastMask, + UnitAttr:$smem_a_read, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CpOp : NVVM_Op<"tcgen05.cp"> { + let description = [{ + Instruction tcgen05.cp initiates an asynchronous copy operation from + shared memory to the location specified by the address operand `taddr` + in the Tensor Memory. The 64-bit register operand `smem_desc` specifies + the matrix descriptor representing the source matrix in the shared memory + that needs to be copied. + + Example: + ```mlir + nvvm.tcgen05.cp %taddr, %smem_desc { + group = #nvvm.cta_group, + shape = #nvvm.tcgen05_cp_shape, + multicast = #nvvm.tcgen05_cp_multicast, + srcFormat = #nvvm.tcgen05_cp_src_fmt + } + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-cp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T64BitSignlessInteger:$smem_desc, + NVVM_Tcgen05CpShapeAttr:$shape, + OptionalAttr:$group, + OptionalAttr:$multicast, + OptionalAttr:$srcFormat + ); + let results = (outs); +} +def NVVM_Tcgen05DeallocOp : NVVM_Op<"tcgen05.dealloc"> { + let description = [{ + The `tcgen05.dealloc` Op de-allocates the tensor core memory + specified by `tmemAddr`, which must be from a previous tensor + memory allocation. The `nCols` operand specifies the number + of columns to be de-allocated, and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05FenceOp : NVVM_Op<"tcgen05.fence"> { + let description = [{ + The `tcgen05.fence` orders all prior async tcgen05 operations + with respect to the subsequent tcgen05 and execution ordering operations. + The `tcgen05.fence` orders all subsequent async tcgen05 operations + with respect to the prior tcgen05 and execution ordering operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence) + }]; + let arguments = (ins + NVVM_Tcgen05FenceKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld"> { + let description = [{ + Instruction `tcgen05.ld` asynchronously loads data from the Tensor Memory at + the location specified by the 32-bit address operand `tmemAddr` into the + destination register `res`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `pack` can be used to pack two 16-bit + elements from adjacent columns into a single 32-bit element during the load. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.ld %tmemAddr, %offset pack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + Optional:$offset, + UnitAttr:$pack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector2:$res + ); +} +def NVVM_Tcgen05LdRedOp : NVVM_Op<"tcgen05.ld.red"> { + let description = [{ + Instruction `tcgen05.ld.red` asynchronously loads data from the Tensor + Memory at the location specified by the 32-bit address operand `addr` into + the destination register `data`, collectively across all threads of the + warp. The operation also performs reduction operation specified by `op` on + the loaded data across columns in each lane and stored into `redVal` + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `addr` + and the base address of the second access is specified by + `addr + offset`, where `offset` is an immediate argument. + + `isSigned` attribute is used to specify whether the integer reduction value is is signed, it is to false by default + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=============================================| + | num/shape | 16x32bx2/32x32b | + |=============================================| + | x2 | 2 | + | x4 | 4 | + | x8 | 8 | + | x16 | 16 | + | x32 | 32 | + | x64 | 64 | + | x128 | 128 | + |=============================================| + ``` + + Example: + ```mlir + // u32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32>, i32 + + // s32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + isSigned = true + } : <2xi32>, i32 + + // f32 + %data, %redval = nvvm.tcgen05.ld,red %addr { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xf32>, f32 + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$addr, + Optional:$offset, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_ReductionKindAttr:$op, + UnitAttr:$abs, + UnitAttr:$nan, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger3:$data, + NVVM_T32BitSignlessIntegerOr32:$redVal + ); +} +def NVVM_Tcgen05LdRedSpcompressOp : NVVM_Op<"tcgen05.ld.red.spcompress"> { + let description = [{ + The `tcgen05.ld.red.spcompress` performs in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + The NaN detection mode is always used for the compression operation, while the .NAN flag only applies + to the reduction portion. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + - `nan`: Whether to use NaN detection mode + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + - `redVal`: Reduction value + + Examples: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // Max operation with absolute value comparison + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with NaN detection for reduction + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<8xi32> + + // MAX operation with abs and nan options + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MIN operation with maximum repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + UnitAttr:$nan, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData, + NVVM_T32BitSignlessInteger:$redVal + ); +} +def NVVM_Tcgen05LdSpcompressOp : NVVM_Op<"tcgen05.ld.spcompress"> { + let description = [{ + The `tcgen05.ld.spcompress` performs an in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + + With a 2-to-4 sparsity factor (specified by `compFactor`), half as much data is written to the output + as is read from Tensor Memory. The sequence of sparse metadata is written to the `metadata` result, + while the compressed data is written to `compressedData`. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + + Example: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // MAX operation with abs option + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with x64 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MAX operation with abs and x128 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData + ); +} +def NVVM_Tcgen05MMAOp : NVVM_Op<"tcgen05.mma", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma` operation is an asynchronous tensor core instruction that + performs matrix multiplication, accumulation in a single fused operation. It + targets 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - `scaleInputD` is an Immediate value operand used for scaling D matrix by 2 ^ (-scaleInputD). The valid range is [0, 15] + + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + - `aShift` shifts the rows of the A matrix down by one row and can only be + applied if A is in tensor memory + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleOp : NVVM_Op<"tcgen05.mma.block_scale"> { + let description = [{ + The `tcgen05.mma.block_scale` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with block scaling in a + single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32 bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute restricted to mxf8f6f4, mxf4, or mxf4nvf4 + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleDecompressBOp : NVVM_Op<"tcgen05.mma.block_scale.decompress_b"> { + let description = [{ + The `tcgen05.mma.block_scale.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation with block scaling in a single fused operation. + It targets 5th-generation tensor cores, providing developers with + fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOpA is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_AnyType:$decompressBMetadata, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMADecompressBOp : NVVM_Op<"tcgen05.mma.decompress_b"> { + let description = [{ + The `tcgen05.mma.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Optional Operands: + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOAp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$decompressBMetadata, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseOp : NVVM_Op<"tcgen05.mma.sp", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma.sp` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with sparse `A` matrix in + a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x (K / 2)` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseBlockScaleOp : NVVM_Op<"tcgen05.mma.sp.block_scale"> { + let description = [{ + The `tcgen05.mma.sp.block_scale` operation is an asynchronous tensor core + instruction that performs matrix multiplication, accumulation with block + scaling, and sparse `A` matrix in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained control + over execution, and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b) // if `enableInputD` is specified + D = (A * scale_a) * (B * scale_b) + D // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + Other attributes and operands are similar to that of tcgen05.mma.block_scale Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$sparseMetadata, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsOp : NVVM_Op<"tcgen05.mma.ws"> { + let description = [{ + The `tcgen05.mma.ws` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + in a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution, and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - zeroColMask is a 64 bit value representing the [Zero-column mask descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-zero-column-mask-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + Default Valued Attributes: + - collectorBBuffer specifies collector buffer for matrix B: b0 (default), b1, b2, b3 + + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsSparseOp : NVVM_Op<"tcgen05.mma.ws.sp"> { + let description = [{ + The `tcgen05.mma.ws.sp` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + with sparse `A` matrix in a single fused operation. It targets 5th-generation + tensor cores, providing developers with fine-grained control over execution, + and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in memory or descriptor format + - B is a K x N matrix + - D is an M x N accumulator matrix + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma.ws Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc` constructs a Shared Memory descriptor + for tcgen05.mma. This descriptor is a 64-bit value which describes the + properties of multiplicand matrix in shared memory including its location + in the shared memory of the current CTA. + + ``` + +-----------+------+------------------------------------------------------+ + | Bit-field | Size | Description | + +-----------+------+------------------------------------------------------+ + | 0-13 | 14 | Matrix start address | + | 14-15 | 2 | Reserved | + | 16-29 | 14 | Leading dim relative-offset (or) absolute-address | + | 30-31 | 2 | Reserved | + | 32-45 | 14 | Stride dimension byte offset | + | 46-48 | 3 | Fixed constant value of 0b001 | + | 49-51 | 3 | Matrix base offset | + | 52 | 1 | Leading dimension stride mode: | + | | | 0: byte offset relative | + | | | 1: byte address absolute | + | 53-60 | 8 | Fixed constant value of 0xb00000000 | + | 61-63 | 3 | Swizzling mode: | + | | | 0: No swizzling | + | | | 1: 128-Byte with 32B atomic swizzling | + | | | 2: 128-Byte swizzling | + | | | 4: 64-Byte swizzling | + | | | 6: 32-Byte swizzling | + | | | (Values 3, 5 and 7 are invalid) | + +-----------+------+------------------------------------------------------+ + ``` + + Example: + ```mlir + %desc = nvvm.tcgen05.mma_smem_desc (%startAddr, %leadingDimOffset, %strideDimOffset, + %baseOffset, %leadingDimMode, %swizzleMode) : (i32, i32, i32, i8, i1, i8) -> i64 + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-shared-memory-descriptor) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddr, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T8BitSignlessInteger2:$swizzleMode + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_Tcgen05RelinquishAllocPermitOp : NVVM_Op<"tcgen05.relinquish_alloc_permit"> { + let description = [{ + The `tcgen05.relinquish_alloc_permit` Op specifies that the CTA + of the executing thread is relinquishing the right to allocate + Tensor Memory. So, it is illegal for a CTA to perform `tcgen05.alloc` + after any of its constituent threads execute `tcgen05.relinquish_alloc_permit`. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05ShiftOp : NVVM_Op<"tcgen05.shift"> { + let description = [{ + The `tcgen05.shift` is an asynchronous instruction which initiates + the shifting of 32-byte elements downwards across all the rows, + except the last, by one row. The operand `taddr` specifies the base + address of the matrix in Tensor Memory whose rows must be down shifted. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-shift) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st"> { + let description = [{ + Instruction `tcgen05.st` asynchronously stores data from the source register `r` + into the Tensor Memory at the location specified by the 32-bit address operand + `tmemAddr`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total dimension of + the data which is stored to the Tensor Memory. The `shape` indicates the base + dimension of data to be accessed. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data that + is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `unpack` can be used to unpack a 32-bit element + in the register into two 16-bit elements and store them in adjacent columns. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.st %tmemAddr, %val, %offset unpack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + NVVM_T32BitSignlessIntegerOrVector2:$val, + Optional:$offset, + UnitAttr:$unpack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs); +} +def NVVM_Tcgen05WaitOp : NVVM_Op<"tcgen05.wait"> { + let description = [{ + The `tcgen05.wait` causes the executing thread to block until + all prior `tcgen05.ld` operations issued by the executing thread + have completed. Similarly, the `tcgen05.wait` causes the executing + thread to block until all prior `tcgen05.st` operations issued by the + executing thread have completed. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait) + }]; + let arguments = (ins + NVVM_Tcgen05WaitKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOpV2 : NVVM_Op<"tcgen05_mma_smem_desc_v2", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc_v2` constructs a Shared Memory descriptor for use as the MMA + Shared Memory operand. It configures the memory layout according to the provided parameters. + + The descriptor contains the following bitfields: + - Bits 63-61 (3 bits): Swizzle descriptor type + 0 = SWIZZLE_NONE matrix descriptor + 1 = SWIZZLE_128B_ATOM_32B + 2 = SWIZZLE_128B matrix descriptor + 3 = RESERVED + 4 = SWIZZLE_64B descriptor + 5 = RESERVED + 6 = SWIZZLE_32B descriptor + 7 = RESERVED + - Bits 60-54 (7 bits): Reserved + - Bit 53 (1 bit): UTCQMMA.LUTB Matrix B K_Segment start offset in K=128 (48B) block + 0: K[0:63] = GDESC.start_address + 0B + 1: K[64:127] = GDESC.start_address + 24B + - Bit 52 (1 bit): Matrix A or B leading dimension mode + 0 = BYTE_OFFSET (legacy) + 1 = NEXT_BUFFER_ADDR (circular buffer) + - Bits 51-49 (3 bits): Matrix A or B base offset (valid only for matrix descriptor 1, 2 or 3, 4) + - Bits 48-46 (3 bits): Matrix A or B descriptor version, needs to be set to 1 + - Bits 45-32 (14 bits): Matrix A or B stride dimension byte offset, 16 byte aligned, 4 LSBs not included + - Bit 31 (1 bit): Reserved + - Bits 30-16 (15 bits): Matrix A or B leading dimension byte offset or next buffer start address + If LEADING_DIM_MODE == BYTE_OFFSET: Matrix A or B leading dimension byte offset + If LEADING_DIM_MODE == NEXT_BUFFER_ADDR: Matrix A or B next buffer start address + Offset or address is 16 byte aligned, 4 LSBs not included + - Bit 15 (1 bit): Reserved + - Bits 14-0 (15 bits): Matrix A or B start address, 16 byte aligned, 4 LSBs not included + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddress, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$descriptorVersion, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T1BitSignlessInteger:$kSegmentOffset, + NVVM_T8BitSignlessInteger2:$swizzleType + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_TensormapCpFenceProxyOp : NVVM_Op<"tensormap.cp_fenceproxy"> { + let description = [{ + The `tensormap.cp_fenceproxy` is a fused copy and fence operation. + Instructions perform the following operations in order: + + Copies data of size specified by the `size` argument, in bytes, from the + location specified by the address operand `src` in shared memory to the + location specified by the address operand `dst` in the global memory, in + the generic proxy. + + Establishes a uni-directional proxy release pattern on the ordering from + the copy operation to the subsequent access performed in the tensormap + proxy on the address `dst` + + The valid value of `size` operand is 128 and must be an immediate. + + The operands `src` and `dst` specify non-generic addresses in shared::cta and + global state space respectively + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dst, + NVVM_LLVMPointerInAddressSpace3:$src, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_TensormapReplaceOp : NVVM_Op<"tensormap.replace"> { + let description = [{ + The `nvvm.tensormap.replace` replaces the specified field of the tensor-map + object at the location specified by `addr` with a new value (specified by + `new_value` or `new_value_attr`). + + The `field` argument specifies the field of the tensor-map object to + replace. + + `new_value` is an `i32`/`i64` argument that specifies the new value to + replace the `field` with for the `global_address`, `rank`, `box_dim`, + `global_dim`, `global_stride`, and `element_stride` fields. It must be an + `i64` for the `global_address` and `global_stride` fields and `i32` for the + remaining fields. + + For `rank`, `new_value` must be one less than the desired tensor rank as + this field uses zero-based numbering. + + `new_value_attr` is an attribute that specifies the new value to replace + the `field` with for the `elemtype`, `interleave_layout`, `swizzle_mode`, + `swizzle_atomicity`, and `fill_mode` fields. It takes the place of + `new_value` for these fields. It must be a valid attribute corresponding to + the `field` type. + + The ordinal `ord` is an immediate integer argument that specifies the + ordinal of the `field` across the tensor which needs to be replaced and is + required only for the `box_dim`, `global_dim`, `global_stride`, and + `element_stride` fields. + + [For more information, see PTX ISA.](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-tensormap-replace) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace14:$addr, + Optional:$new_value, + NVVM_TensormapFieldAttr:$field, + OptionalAttr:$ord, + OptionalAttr:$new_value_attr + ); + let results = (outs); +} +def NVVM_TraceMarkOp : NVVM_Op<"trace.mark", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.trace.mark` operation introduces trace events for profiling + programs. + + It takes `event_type`, a 12-bit value in the range 0-4095, which denotes + the type of the event. Additionally, it takes `domain` and `event`, which + are strings that identify the domain and event name of the trace event. + Finally, it takes an optional 32-bit or 64-bit payload, an optional + payload descriptor (required when payload is provided), and a predicate to + conditionally execute the trace event. + + Examples: + + ```mlir + // Basic trace event + nvvm.trace.mark event_type = 42, domain = "myDomain", event = "myEvent" + + // With 32-bit payload + nvvm.trace.mark event_type = 42, payload = %p, + domain = "myDomain", event = "myEvent", payload_descriptor = "counter" + : i32 + + // Conditional trace event + nvvm.trace.mark event_type = 42, predicate = %pred, + domain = "myDomain", event = "myEvent" + ``` + }]; + let arguments = (ins + Optional:$payload, + Optional:$predicate, + I32Attr:$eventType, + StrAttr:$domain, + StrAttr:$event, + OptionalAttr:$payloadDescriptor + ); + let results = (outs); +} +def NVVM_VoteSyncOp : NVVM_InferredOp<"vote.sync"> { + let description = [{ + The `vote.sync` op will cause executing thread to wait until all non-exited + threads corresponding to membermask have executed `vote.sync` with the same + qualifiers and same membermask value before resuming execution. + + The vote operation kinds are: + - `any`: True if source predicate is True for some thread in membermask. + - `all`: True if source predicate is True for all non-exited threads in + membermask. + - `uni`: True if source predicate has the same value in all non-exited + threads in membermask. + - `ballot`: In the ballot form, the destination result is a 32 bit integer. + In this form, the predicate from each thread in membermask are copied into + the corresponding bit position of the result, where the bit position + corresponds to the thread's lane id. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-vote-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask, + NVVM_T1BitSignlessInteger:$pred, + NVVM_VoteSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr1:$res + ); +} +def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned"> { + let description = [{ + Commits all prior uncommitted warpgroup level matrix multiplication operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned"> { + let description = [{ + Enforce an ordering of register accesses between warpgroup level matrix + multiplication and other operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-fence) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async"> { + let description = [{ + The warpgroup (128 threads) level matrix multiply and accumulate operation + has either of the following forms, where matrix D is called accumulator: + D = A * B + D + D = A * B, where the input from accumulator D is disabled. + + Supported shapes: + ``` + |--------------|--------------|------------|--------------|---------------| + | | | | |f16+=e4m3*e4m3 | + | | | | |f16+=e5m2*e5m2 | + |f32+=tf32*tf32|f16+=f16 *f16 | s32+=s8*s8 |s32 += b1 * b1|f16+=e5m2*e4m3 | + | |f32+=f16 *f16 | s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=s8*u8 | |f32+=e4m3*e4m3 | + | | | s32+=u8*s8 | |f32+=e5m2*e5m2 | + | | | | |f32+=e4m3*e5m2 | + | | | | |f32+=e4m3*e5m2 | + |--------------|--------------|------------|--------------|---------------| + | .m64n8k8 | .m64n8k16 | .m64n8k32 | .m64n8k256 | .m64n8k32 | + | .m64n16k8 | .m64n16k16 | .m64n16k32 | .m64n16k256 | .m64n16k32 | + | .m64n24k8 | .m64n24k16 | .m64n24k32 | .m64n24k256 | .m64n24k32 | + | .m64n32k8 | .m64n32k16 | .m64n32k32 | .m64n32k256 | .m64n32k32 | + | .m64n40k8 | .m64n40k16 | .m64n48k32 | .m64n48k256 | .m64n40k32 | + | .m64n48k8 | .m64n48k16 | .m64n64k32 | .m64n64k256 | .m64n48k32 | + | .m64n56k8 | .m64n56k16 | .m64n80k32 | .m64n80k256 | .m64n56k32 | + | .m64n64k8 | .m64n64k16 | .m64n96k32 | .m64n96k256 | .m64n64k32 | + | .m64n72k8 | .m64n72k16 | .m64n112k32| .m64n112k256 | .m64n72k32 | + | .m64n80k8 | .m64n80k16 | .m64n128k32| .m64n128k256 | .m64n80k32 | + | .m64n88k8 | .m64n88k16 | .m64n144k32| .m64n144k256 | .m64n88k32 | + | .m64n96k8 | .m64n96k16 | .m64n160k32| .m64n160k256 | .m64n96k32 | + | .m64n104k8 | .m64n104k16 | .m64n176k32| .m64n176k256 | .m64n104k32 | + | .m64n112k8 | .m64n112k16 | .m64n192k32| .m64n192k256 | .m64n112k32 | + | .m64n120k8 | .m64n120k16 | .m64n208k32| .m64n208k256 | .m64n120k32 | + | .m64n128k8 | .m64n128k16 | .m64n224k32| .m64n224k256 | .m64n128k32 | + | .m64n136k8 | .m64n136k16 | .m64n240k32| .m64n240k256 | .m64n136k32 | + | .m64n144k8 | .m64n144k16 | .m64n256k32| .m64n256k256 | .m64n144k32 | + | .m64n152k8 | .m64n152k16 | | | .m64n152k32 | + | .m64n160k8 | .m64n160k16 | | | .m64n160k32 | + | .m64n168k8 | .m64n168k16 | | | .m64n168k32 | + | .m64n176k8 | .m64n176k16 | | | .m64n176k32 | + | .m64n184k8 | .m64n184k16 | | | .m64n184k32 | + | .m64n192k8 | .m64n192k16 | | | .m64n192k32 | + | .m64n200k8 | .m64n200k16 | | | .m64n200k32 | + | .m64n208k8 | .m64n208k16 | | | .m64n208k32 | + | .m64n216k8 | .m64n216k16 | | | .m64n216k32 | + | .m64n224k8 | .m64n224k16 | | | .m64n224k32 | + | .m64n232k8 | .m64n232k16 | | | .m64n232k32 | + | .m64n240k8 | .m64n240k16 | | | .m64n240k32 | + | .m64n248k8 | .m64n248k16 | | | .m64n248k32 | + | .m64n256k8 | .m64n256k16 | | | .m64n256k32 | + |--------------|--------------|------------|--------------|---------------| + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions) + }]; + let arguments = (ins + NVVM_LLVMStructureType:$inouts, + NVVM_T64BitSignlessInteger:$descriptorA, + NVVM_T64BitSignlessInteger:$descriptorB, + NVVM_MMAShapeAttr:$shape, + NVVM_WGMMATypesAttr:$typeA, + NVVM_WGMMATypesAttr:$typeB, + NVVM_WGMMATypesAttr:$typeD, + NVVM_WGMMAScaleOutAttr:$scaleD, + NVVM_WGMMAScaleInAttr:$scaleA, + NVVM_WGMMAScaleInAttr:$scaleB, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$satfinite + ); + let results = (outs + NVVM_LLVMStructureType:$results_ + ); +} +def NVVM_WgmmaWaitGroupSyncOp : NVVM_Op<"wgmma.wait.group.sync.aligned"> { + let description = [{ + Signal the completion of a preceding warpgroup operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group) + }]; + let arguments = (ins + I64Attr:$group + ); + let results = (outs); +} +def NVVM_WMMALoadOp : NVVM_Op<"wmma.load"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype, + NVVM_MMAFragAttr:$frag + ); + let results = (outs + NVVM_LLVMStructureTypeOr64Bit:$res + ); +} +def NVVM_WMMAMmaOp : NVVM_Op<"wmma.mma"> { + let arguments = (ins + Variadic:$args, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$eltypeA, + NVVM_MMATypesAttr:$eltypeB + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_WMMAStoreOp : NVVM_Op<"wmma.store"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + Variadic:$args, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype + ); + let results = (outs); +} + +#endif // MLIR_DIALECTS_NVVM_NVVMOPS_TD diff --git a/mlir/dialects/nvvm/capi.cc b/mlir/dialects/nvvm/capi.cc new file mode 100644 index 00000000..ff71af81 --- /dev/null +++ b/mlir/dialects/nvvm/capi.cc @@ -0,0 +1,1964 @@ +//===- capi.cc - C API for CuTe's nvvm dialect ----------------------------===// + +#include "capi.h" + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm, + mlir::cutlass_compiler::nvvm::NVVMDialect) + +// Attributes + +namespace nvvm = mlir::cutlass_compiler::nvvm; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::AtomicOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReductionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BlockScaleFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CTAGroupKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CVTPackFloatKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CacheEvictionPriorityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType( + value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ClusterLaunchControlQueryTypeAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CompareOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP4TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP8TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::DotAccumulateTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeEvictKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::EvictKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::FPRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::GridDepActionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::IntegerRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::L2PrefetchSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LdStMatrixEltTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierExtKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierPhaseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierSpaceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierTxnKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierWaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAB1Op(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAB1OpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMABlockScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMACtaCount(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMACtaCountAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAFrag(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAFragAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMALayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMALayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MatchSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemOrderKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMulMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MulModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::NVVMMemorySpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePermuteMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PermuteModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PrefetchCacheLevelAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeProxyKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ProxyKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ScaleVecSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SetMaxRegisterActionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSharedSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SharedSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeShflKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ShflKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SparsityFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStateSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StateSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTCBarParam(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TCBarParamAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMALoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMALoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpMulticastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05FenceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05LdStShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMABlockScaleAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer( + value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorBBufferAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05WaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapElemtypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFieldAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFillModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapInterleaveLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleAtomicityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTmemLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TmemLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ValidatePatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::VoteSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleInAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleOutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, + int n) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::LdStMatrixShapeAttr::get(ctx, m, n)); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} + +bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, + int k) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::MMAShapeAttr::get(ctx, m, n, k)); +} +int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} +int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getK(); +} + +bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, + MlirStringRef triple, + MlirStringRef chip, + MlirStringRef features, + MlirAttribute flags, MlirAttribute link, + bool verifyTarget) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr flagsValue; + if (!unwrapAs(ctx, flags, true, flagsValue)) + return {nullptr}; + ::mlir::ArrayAttr linkValue; + if (!unwrapAs(ctx, link, true, linkValue)) + return {nullptr}; + return wrap(nvvm::TargetAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, + unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, + verifyTarget)); +} +int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} +MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getTriple()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getFeatures()); +} +MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFlags())); +} +MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLink())); +} +bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getVerifyTarget(); +} diff --git a/mlir/dialects/nvvm/capi.h b/mlir/dialects/nvvm/capi.h new file mode 100644 index 00000000..ee604770 --- /dev/null +++ b/mlir/dialects/nvvm/capi.h @@ -0,0 +1,785 @@ +//===- capi.h - C API for CuTe's nvvm dialect ---------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_NVVM_CAPI_H +#define MLIR_DIALECTS_NVVM_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#nvvm.atomic_op`: operations supported by atom instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction`: NVVM barrier reduction operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); + +// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_format`: MMA Block Scale Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.cta_group`: NVVM CTA group kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); + +// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.op`: Comparison operator encoding +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.grid_dep_action`: Action kind for grid dependency control +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_shape`: shape attribute for ldmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_src_format`: source format for ldmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_space`: NVVM MBarrier space kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_b1op`: MMA binary operations +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_kind`: Block Scale Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_cta_count`: MMA CTA count +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_frag`: NVVM MMA frag type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_int_overflow`: MMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_kind`: MMA operation kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_layout`: NVVM MMA layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_type`: NVVM MMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.match_sync_kind`: NVVM match sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_order`: NVVM Memory Ordering kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_scope`: NVVM Memory Scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mul_mode`: multiply mode attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.memory_space`: NVVM Memory Space +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.permute_mode`: NVVM permute mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); + +// `#nvvm.proxy_kind`: Proxy kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_op`: Ops supported by red instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_type`: types supported by red instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_op_kind`: spcompress operation kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat_mode`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.action`: NVVM set max register action +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); + +// `#nvvm.shared_space`: Shared memory space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.shfl_kind`: NVVM shuffle kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.sparsity_format`: MMA Sparsity Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.state_space`: NVVM State Space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_shape`: shape attribute for stmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, +// 3, 4 +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.validate_pattern`: NVVM validate data pattern +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); + +// `#nvvm.vote_sync_kind`: NVVM vote sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_in`: WGMMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_out`: WGMMA input predicate +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_type`: NVVM WGMMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); + +// `#nvvm.shape`: Shape of an MMA operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); + +// `#nvvm.target`: GPU target of an NVVM module +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet( + MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirAttribute flags, MlirAttribute link, + bool verifyTarget); +MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_NVVM_CAPI_H diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig new file mode 100644 index 00000000..14a0e8e2 --- /dev/null +++ b/mlir/dialects/nvvm/nvvm.zig @@ -0,0 +1,311 @@ +//! Builders for the NVVM operations CuTe kernels emit, over the bindings of +//! NVIDIA's `nvvm` dialect (`mlir/dialects/nvvm`, `ops`): the CuTe-DSL +//! compiler's own NVVM, which the upstream dialect differs from. Each builder +//! fixes the attributes the kernels do not choose. + +const std = @import("std"); + +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvvm"); +} + +fn enumAttribute(comptime T: type, ctx: *mlir.Context, value: @FieldType(T.InitArgs, "value")) *const mlir.Attribute { + return (T.get(ctx, .{ .value = value }) catch unreachable).attribute(); +} + +/// `#nvvm.cta_group<...>`: CTAs cooperating on a tcgen05 operation. +pub const CtaGroup = ops.CTAGroupKind; +/// `#nvvm.tcgen05_fence<...>`: `tcgen05.fence::{before,after}_thread_sync`. +pub const Tcgen05FenceKind = ops.Tcgen05FenceKind; +/// `#nvvm.tcgen05_wait<...>`: `tcgen05.wait::{ld,st}`. +pub const Tcgen05WaitKind = ops.Tcgen05WaitKind; +/// `#nvvm.tcgen05_mma_kind<...>`: input types of `tcgen05.mma`. +pub const Tcgen05MmaKind = ops.Tcgen05MMAKind; +/// `#nvvm.tcgen05_ldst_shape<...>`: lane/bit shape of `tcgen05.ld` / `tcgen05.st`. +pub const Tcgen05LdStShape = ops.Tcgen05LdStShape; +/// `#nvvm.mem_scope<...>`: scope of a memory operation. +pub const MemScope = ops.MemScopeKind; +/// `#nvvm.mbar_wait<...>`: `test` (non-blocking) or `try` (may suspend) wait. +pub const MBarrierWaitKind = ops.MBarrierWaitKind; +/// `#nvvm.mbar_scope<...>`: scope of an mbarrier wait. +pub const MBarrierScope = ops.MBarrierScopeKind; +/// `#nvvm.proxy_kind<...>`: memory proxy of `fence.proxy`. +pub const ProxyKind = ops.ProxyKind; +/// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. +pub const SharedSpace = ops.SharedSpace; + +// ============================================================================= +// Kernel attributes +// ============================================================================= + +/// `nvvm.reqntid`: the CTA shape (x, y, z) a kernel is always launched with. +pub fn reqntid(ctx: *mlir.Context, dims: [3]i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.reqntid", .denseArray(ctx, .i32, &dims)); +} + +/// `nvvm.minctasm`: the number of CTAs a kernel must fit on one SM at once. +pub fn minctasm(ctx: *mlir.Context, ctas: i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.minctasm", .int(ctx, .i32, ctas)); +} + +// ============================================================================= +// Special registers and thread synchronization +// ============================================================================= + +/// nvvm.read.ptx.sreg. — an i32 special register, e.g. `tid.x`, `ctaid.y`, +/// `cluster.ctarank`. +pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const mlir.Location) *mlir.Operation { + const builder = comptime blk: { + var fn_name: [name.len]u8 = name[0..name.len].*; + std.mem.replaceScalar(u8, &fn_name, '.', '_'); + break :blk "read_ptx_sreg_" ++ fn_name; + }; + return @field(ops, builder)(ctx, .int(ctx, .i32), .{}, location); +} + +/// nvvm.barrier — `bar.sync 0` over the whole CTA. +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.barrier(ctx, .{}, location); +} + +/// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.bar_warp_sync(ctx, mask, location); +} + +/// nvvm.elect.sync — i1, true in one elected lane of the warp. +pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.elect_sync(ctx, .int(ctx, .i1), .{}, location); +} + +// ============================================================================= +// mbarrier +// ============================================================================= + +/// nvvm.mbarrier.init — initialize the mbarrier at `ptr` (`!llvm.ptr<3>`) to expect +/// `count` (i32) arrivals per phase. +pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_init(ctx, ptr, count, .{ .layout = enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0) }, location); +} + +/// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). +pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_arrive( + ctx, + ptr, + .{ .count = count, .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); +} + +/// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous +/// transactions (SM90+). +pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_arrive_expect_tx( + ctx, + ptr, + bytes, + .{ .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); +} + +/// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed +/// (`test`: never suspends; `try`: may suspend for a system-dependent time). +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + kind: MBarrierWaitKind, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_wait_parity( + ctx, + ptr, + parity, + .int(ctx, .i1), + enumAttribute(ops.MBarrierWaitKindAttr, ctx, kind), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope) }, + location, + ); +} + +/// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) +/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) +/// is a suspend-time hint in nanoseconds per try. +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); +} + +/// One potentially suspending parity test with acquire semantics. A false +/// result means the timeout expired; callers that need completion must retry. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity_timelimit( + ctx, + ptr, + parity, + suspend_time, + .int(ctx, .i1), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, + location, + ); +} + +/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.fence_mbarrier_init(ctx, location); +} + +/// nvvm.fence.proxy — order memory accesses across proxies. +pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { + return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), .{ .space = enumAttribute(ops.SharedSpaceAttr, ctx, space) }, location); +} + +// ============================================================================= +// Asynchronous copies +// ============================================================================= + +/// nvvm.cp.async.commit.group — commit the pending `cp.async` copies as a group. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_commit_group(ctx, location); +} + +/// nvvm.cp.async.wait.group — wait until at most `pending` `cp.async` groups are in flight. +pub fn cp_async_wait_group(ctx: *mlir.Context, pending: u32, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_wait_group(ctx, .int(ctx, .i32, pending), location); +} + +/// nvvm.cp.async.bulk.commit.group — commit the pending bulk (TMA) copies as a group. +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_bulk_commit_group(ctx, location); +} + +/// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in +/// flight; with `read`, only until their sources have been read. +pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), .{ .read = if (read) .unit(ctx) else null }, location); +} + +// ============================================================================= +// tcgen05 (SM100 tensor cores and tensor memory) +// ============================================================================= + +/// nvvm.tcgen05.alloc — allocate `columns` (i32) tensor-memory columns and write the +/// address to `holder` (`!llvm.ptr<3>`). +pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_alloc(ctx, holder, columns, .{}, location); +} + +/// nvvm.tcgen05.dealloc — free `columns` (i32) tensor-memory columns at `taddr` +/// (`!llvm.ptr<6>`). +pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_dealloc(ctx, taddr, columns, .{}, location); +} + +/// nvvm.tcgen05.fence — order tcgen05 operations around a thread synchronization. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: Tcgen05FenceKind, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_fence(ctx, enumAttribute(ops.Tcgen05FenceKindAttr, ctx, kind), location); +} + +/// nvvm.tcgen05.wait — wait for the issued tensor-memory loads or stores. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); +} + +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the +/// tcgen05 MMA/copy operations issued by the executing thread with `group` +/// complete. An optional mask selects CTAs in the cluster whose barriers at +/// the same shared-memory offset receive one arrival each. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +} + +/// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory +/// descriptor, i64) * B (descriptor, i64) [+ D when `accumulate` (i1)], with the +/// instruction descriptor `idesc` (i32). +pub fn tcgen05_mma( + ctx: *mlir.Context, + d: *const mlir.Value, + a_desc: *const mlir.Value, + b_desc: *const mlir.Value, + idesc: *const mlir.Value, + accumulate: *const mlir.Value, + kind: Tcgen05MmaKind, + cta_group: CtaGroup, + location: *const mlir.Location, +) *mlir.Operation { + return ops.tcgen05_mma( + ctx, + d, + a_desc, + b_desc, + idesc, + accumulate, + enumAttribute(ops.Tcgen05MMAKindAttr, ctx, kind), + enumAttribute(ops.CTAGroupKindAttr, ctx, cta_group), + .{}, + location, + ); +} + +/// nvvm.tcgen05.ld — load `result_type` (an i32 or vector of i32) from tensor memory +/// at `taddr` (`!llvm.ptr<6>`). +pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_ld(ctx, taddr, result_type, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); +} + +/// nvvm.tcgen05.st — store `value` (an i32 or vector of i32) to tensor memory at `taddr`. +pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_st(ctx, taddr, value, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); +} + +// ============================================================================= +// Clusters +// ============================================================================= + +/// nvvm.mapa — the address of `ptr` in the shared memory of cluster CTA `rank` (i32). +pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return ops.mapa(ctx, ptr, rank, result_type, location); +} + +/// nvvm.cluster.arrive — arrive on the cluster barrier. +pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cluster_arrive(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); +} + +/// nvvm.cluster.wait — wait on the cluster barrier. +pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cluster_wait(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); +} + +// ============================================================================= +// Math +// ============================================================================= + +/// nvvm.ex2 — `ex2.approx[.ftz].f32`. +pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { + return ops.ex2(ctx, value, .float(ctx, .f32), .{ .ftz = .boolean(ctx, ftz) }, location); +} + +/// nvvm.rcp.approx.ftz.f — `rcp.approx.ftz.f32`. +pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.rcp_approx_ftz_f(ctx, value, .float(ctx, .f32), location); +} diff --git a/mlir/dialects/nvvm/ops.zig b/mlir/dialects/nvvm/ops.zig new file mode 100644 index 00000000..88c1c514 --- /dev/null +++ b/mlir/dialects/nvvm/ops.zig @@ -0,0 +1,8389 @@ +//! Zig bindings of NVIDIA's NVVM dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// The C dialect handle, `mlirGetDialectHandle__cute_nvvm__`. +pub const dialect_handle = "cute_nvvm"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// operations supported by atom instruction. +pub const AtomicOpKind = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + cas = 3, + exch = 4, + add = 5, + inc = 6, + dec = 7, + min = 8, + max = 9, + umin = 10, + umax = 11, + fadd = 12, + sub = 13, +}; + +/// NVVM barrier reduction operation. +pub const BarrierReduction = enum(u32) { + popc = 0, + @"and" = 1, + @"or" = 2, +}; + +/// NVVM barrier redux kind. +pub const BarrierReduxKind = enum(u32) { + @"and" = 0, + @"or" = 1, + popc = 2, +}; + +/// MMA Block Scale Format. +pub const BlockScaleFormat = enum(u32) { + ue8m0 = 0, + ue4m3 = 1, +}; + +/// NVVM CTA group kind. +pub const CTAGroupKind = enum(u32) { + cta_1 = 0, + cta_2 = 1, +}; + +/// NVVM CVT Pack Float kind. +pub const CVTPackFloatKind = enum(u32) { + f32 = 0, + f16x2 = 1, + e4m3x2 = 2, + e5m2x2 = 3, + bf16x2 = 4, + e2m1x2 = 5, + e2m3x2 = 6, + e3m2x2 = 7, + ue8m0x2 = 8, +}; + +/// NVVM Cache Eviction Priority. +pub const CacheEvictionPriority = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + no_allocate = 4, +}; + +/// NVVM ClusterLaunchControlQueryType. +pub const ClusterLaunchControlQueryType = enum(u32) { + is_canceled = 0, + get_first_cta_id_x = 1, + get_first_cta_id_y = 2, + get_first_cta_id_z = 3, +}; + +/// Comparison operator encoding. +pub const CompareOpKind = enum(u32) { + eq = 0, + ne = 1, + ugt = 2, + uge = 3, + ult = 4, + ule = 5, + sgt = 6, + sge = 7, + slt = 8, + sle = 9, +}; + +/// NVVM ConvertFP4Type kind. +pub const ConvertFP4Type = enum(u32) { + e2m1 = 0, +}; + +/// NVVM ConvertFP8Type kind. +pub const ConvertFP8Type = enum(u32) { + e4m3 = 0, + e5m2 = 1, + ue8m0 = 2, + ue5m3 = 3, +}; + +/// NVVM ConvertScale kind. +pub const ConvertScaleKind = enum(u32) { + none = 0, + ue8m0 = 1, +}; + +/// NVVM DotAccumulateType. +pub const DotAccumulateType = enum(u32) { + signed = 1, + unsigned = 0, +}; + +/// NVVM L2 Prefetch Size. +pub const EvictKind = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + evict_no_allocate = 4, + none = 5, +}; + +/// NVVM FPRoundingMode kind. +pub const FPRoundingMode = enum(u32) { + none = 0, + rn = 1, + rm = 2, + rp = 3, + rz = 4, + rna = 5, + rs = 6, +}; + +/// Action kind for grid dependency control. +pub const GridDepActionKind = enum(u32) { + wait = 0, + launch_dependents = 1, +}; + +/// NVVM IntegerRoundingMode kind. +pub const IntegerRoundingMode = enum(u32) { + none = 0, + rni = 1, + rzi = 2, + rmi = 3, + rpi = 4, +}; + +/// NVVM L2 Prefetch Size. +pub const L2PrefetchSize = enum(u32) { + none = 0, + reserved = 1, + size_64b = 2, + size_128b = 3, + size_256b = 4, +}; + +/// Element type for ldmatrix and stmatrix. +pub const LdStMatrixEltType = enum(u32) { + b16 = 0, + b8 = 1, + @"b8x16.b6x16_p32" = 2, + @"b8x16.b4x16_p64" = 3, +}; + +/// NVVM load cache modifier kind(Ext). +pub const LoadCacheModifierExtKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, + none = 5, +}; + +/// NVVM load cache modifier kind. +pub const LoadCacheModifierKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, +}; + +/// shape attribute for ldmatrix. +pub const LoadShape = enum(u32) { + m8n8 = 0, + m8n16 = 1, + m16n16 = 3, +}; + +/// source format for ldmatrix. +pub const LoadSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, + b8 = 2, +}; + +/// NVVM MBarrier Layout. +pub const MBarrierLayout = enum(u32) { + v0 = 0, + v1 = 1, +}; + +/// NVVM mbarrier phase type. +pub const MBarrierPhase = enum(u32) { + none = 0, + primary = 1, + conditional = 2, +}; + +/// NVVM MBarrier scope kind. +pub const MBarrierScopeKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier space kind. +pub const MBarrierSpaceKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier Transaction kind. +pub const MBarrierTxnKind = enum(u32) { + arrive = 0, + arrive_drop = 1, + arrive_expect_tx = 2, + arrive_drop_expect_tx = 3, + expect_tx = 4, + complete_tx = 5, +}; + +/// NVVM MBarrier wait kind. +pub const MBarrierWaitKind = enum(u32) { + @"test" = 0, + @"try" = 1, +}; + +/// MMA binary operations. +pub const MMAB1Op = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Block Scale Kind. +pub const MMABlockScaleKind = enum(u32) { + mxf8f6f4 = 0, + mxf4 = 1, + mxf4nvf4 = 2, +}; + +/// MMA CTA count. +pub const MMACtaCount = enum(u32) { + cta1 = 1, + cta2 = 2, +}; + +/// NVVM MMA frag type. +pub const MMAFrag = enum(u32) { + a = 0, + b = 1, + c = 2, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// MMA operation kind. +pub const MMAKind = enum(u32) { + f8f6f4 = 0, +}; + +/// NVVM MMA layout. +pub const MMALayout = enum(u32) { + row = 0, + col = 1, +}; + +/// NVVM MMA types. +pub const MMATypes = enum(u32) { + f16 = 0, + f32 = 1, + tf32 = 2, + bf16 = 9, + s8 = 4, + u8 = 3, + s32 = 5, + s4 = 8, + u4 = 7, + b1 = 6, + f64 = 10, + e4m3 = 11, + e5m2 = 12, + e3m2 = 13, + e2m3 = 14, + e2m1 = 15, +}; + +/// NVVM match sync kind. +pub const MatchSyncKind = enum(u32) { + any = 0, + all = 1, +}; + +/// NVVM Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + weak = 0, + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, + sc = 5, + mmio = 6, + @"volatile" = 8, + constant = 7, +}; + +/// NVVM Memory Scope kind. +pub const MemScopeKind = enum(u32) { + cta = 0, + cluster = 1, + gpu = 2, + sys = 3, +}; + +/// multiply mode attribute. +pub const MulMode = enum(u32) { + hi = 0, + lo = 1, + wide = 2, +}; + +/// NVVM Memory Space. +pub const NVVMMemorySpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + tensor = 6, + shared_cluster = 7, +}; + +/// NVVM permute mode. +pub const PermuteMode = enum(u32) { + default = 0, + f4e = 1, + b4e = 2, + rc8 = 3, + ecl = 4, + ecr = 5, + rc16 = 6, +}; + +/// NVVM Prefetch Cache Level. +pub const PrefetchCacheLevel = enum(u32) { + L1 = 0, + L2 = 1, +}; + +/// Proxy kind. +pub const ProxyKind = enum(u32) { + alias = 0, + async = 1, + @"async.global" = 2, + @"async.shared" = 3, + tensormap = 4, + generic = 5, +}; + +/// NVVM Reduction Kind attribute. +pub const ReductionKind = enum(u32) { + add = 1, + @"and" = 2, + max = 3, + min = 4, + @"or" = 5, + umax = 6, + umin = 7, + xor = 8, + fmin = 9, + fmax = 10, +}; + +/// Ops supported by red instruction. +pub const ReductionOp = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + add = 3, + inc = 4, + dec = 5, + min = 6, + max = 7, +}; + +/// types supported by red instruction. +pub const ReductionType = enum(u32) { + b32 = 0, + b64 = 1, + u32 = 2, + u64 = 3, + s32 = 4, + s64 = 5, + f32 = 6, + f64 = 7, + f16 = 8, + f16x2 = 9, + bf16 = 10, + bf16x2 = 11, +}; + +/// Sparse tensor compression element size. +pub const SPCompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor compression factor type. +pub const SPCompressFactorType = enum(u32) { + sp2to4 = 0, +}; + +/// Sparse tensor compression index size. +pub const SPCompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// spcompress operation kind. +pub const SPCompressOpKind = enum(u32) { + min = 0, + max = 1, +}; + +/// Sparse tensor compression repetition factor. +pub const SPCompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, + x128 = 128, +}; + +/// Sparse tensor decompression element size. +pub const SPDecompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor decompression factor type. +pub const SPDecompressFactorType = enum(u32) { + sp1to2 = 0, + sp1to4 = 1, + sp1to8 = 2, + sp1to16 = 3, + sp2to4 = 4, + sp2to8 = 5, + sp2to16 = 6, + sp4to8 = 7, + sp4to16 = 8, + sp8to16 = 9, +}; + +/// Sparse tensor decompression index size. +pub const SPDecompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// Sparse tensor decompression repetition factor. +pub const SPDecompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, +}; + +/// NVVM SaturationMode kind. +pub const SaturationMode = enum(u32) { + none = 0, + satfinite = 1, + sat = 2, +}; + +/// NVVM SaturationMode kind. +pub const SaturationModeKind = enum(u32) { + none = 0, + satfinite = 1, +}; + +/// MMA Scale Vector Sizes. +pub const ScaleVecSize = enum(u32) { + x1 = 0, + x2 = 1, + x4 = 2, +}; + +/// NVVM set max register action. +pub const SetMaxRegisterAction = enum(u32) { + decrease = 1, + increase = 0, +}; + +/// Shared memory space. +pub const SharedSpace = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM shuffle kind. +pub const ShflKind = enum(u32) { + bfly = 0, + up = 1, + down = 2, + idx = 3, +}; + +/// MMA Sparsity Format. +pub const SparsityFormat = enum(u32) { + thread = 0, +}; + +/// NVVM State Space. +pub const StateSpace = enum(u32) { + generic = 0, + global = 1, + shared_cta = 2, + shared_cluster = 3, + constant = 4, + local = 5, + tensor = 6, +}; + +/// NVVM store cache modifier kind. +pub const StoreCacheModifierKind = enum(u32) { + wb = 0, + cg = 1, + cs = 2, + wt = 3, + none = 4, +}; + +/// shape attribute for stmatrix. +pub const StoreShape = enum(u32) { + m8n8 = 0, + m16n8 = 2, +}; + +/// Cluster MMA Barrier Parameter Type. +pub const TCBarParam = enum(u32) { + a1t0 = 0, + a0tx = 1, +}; + +/// NVVM TMA Load Mode. +pub const TMALoadMode = enum(u32) { + tile = 0, + im2col = 1, + im2col_w = 2, + im2col_w_128 = 3, + tile_gather4 = 4, +}; + +/// NVVM TMA redux kind. +pub const TMAReduxKind = enum(u32) { + add = 0, + max = 2, + min = 1, + inc = 3, + dec = 4, + @"and" = 5, + @"or" = 6, + xor = 7, +}; + +/// NVVM TMA Store Mode. +pub const TMAStoreMode = enum(u32) { + tile = 0, + im2col = 1, + tile_scatter4 = 2, + im2col_w = 3, +}; + +/// tcgen05 cp multicast. +pub const Tcgen05CpMulticast = enum(u32) { + none = 0, + warpx2_02_13 = 1, + warpx2_01_23 = 2, + warpx4 = 3, +}; + +/// tcgen05 cp shapes. +pub const Tcgen05CpShape = enum(u32) { + shape_128x256b = 0, + shape_4x256b = 1, + shape_128x128b = 2, + shape_64x128b = 3, + shape_32x128b = 4, +}; + +/// tcgen05 cp source format. +pub const Tcgen05CpSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, +}; + +/// NVVM Tcgen05 fence kind. +pub const Tcgen05FenceKind = enum(u32) { + before = 0, + after = 1, +}; + +/// allowed 32-bit signless integer cases: 0, 1, 2, 3, 4. +pub const Tcgen05LdStShape = enum(u32) { + shape_16x64b = 0, + shape_16x128b = 1, + shape_16x256b = 2, + shape_32x32b = 3, + shape_16x32bx2 = 4, +}; + +/// tcgen05.mma block scale attribute. +pub const Tcgen05MMABlockScale = enum(u32) { + default = 0, + block16 = 1, + block32 = 2, +}; + +/// tcgen05 MMA Collector Buffer B Attribute. +pub const Tcgen05MMACollectorBBuffer = enum(u32) { + b0 = 0, + b1 = 1, + b2 = 2, + b3 = 3, +}; + +/// tcgen05.mma Collector Buffer Operation. +pub const Tcgen05MMACollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// tcgen05 MMA Supported Types. +pub const Tcgen05MMAKind = enum(u32) { + f16 = 0, + tf32 = 1, + f8f6f4 = 2, + i8 = 3, + mxf8f6f4 = 4, + mxf4 = 5, + mxf4nvf4 = 6, + ti16 = 7, +}; + +/// NVVM Tcgen05 wait kind. +pub const Tcgen05WaitKind = enum(u32) { + load = 0, + store = 1, +}; + +/// NVVM Tensormap Elemtype. +pub const TensormapElemtype = enum(u32) { + u8 = 0, + u16 = 1, + u32 = 2, + s32 = 3, + u64 = 4, + s64 = 5, + f16 = 6, + f32 = 7, + @"f32.ftz" = 8, + f64 = 9, + bf16 = 10, + tf32 = 11, + @"tf32.ftz" = 12, + b4x16 = 13, + b4x16_p64 = 14, + b6x16_p32 = 15, +}; + +/// NVVM Tensormap Field Kind. +pub const TensormapField = enum(u32) { + global_address = 0, + rank = 1, + box_dim = 2, + global_dim = 3, + global_stride = 4, + element_stride = 5, + elemtype = 6, + interleave_layout = 7, + swizzle_mode = 8, + swizzle_atomicity = 9, + fill_mode = 10, +}; + +/// NVVM Tensormap Fill Mode. +pub const TensormapFillMode = enum(u32) { + zero = 0, + oob_nan = 1, +}; + +/// NVVM Tensormap Interleave Layout. +pub const TensormapInterleaveLayout = enum(u32) { + no_interleave = 0, + b16 = 1, + b32 = 2, +}; + +/// NVVM Tensormap Swizzle Atomicity. +pub const TensormapSwizzleAtomicity = enum(u32) { + b16 = 0, + b32 = 1, + b32_flip_b8 = 2, + b64 = 3, +}; + +/// NVVM Tensormap Swizzle Mode. +pub const TensormapSwizzleMode = enum(u32) { + no_swizzling = 0, + b32 = 1, + b64 = 2, + b128 = 3, + b96 = 4, +}; + +/// Tensor Memory Layout Enumerated Type. +pub const TmemLayout = enum(u32) { + tmem_16dp_128bit = 0, + tmem_16dp_256bit = 1, + tmem_32dp_32bit = 2, + tmem_16dp_64bit = 3, + tmem_16dp_32bit_t0_t15 = 4, + tmem_16dp_32bit_t16_t31 = 5, +}; + +/// NVVM validate data pattern. +pub const ValidatePattern = enum(u32) { + none = 0, + per_16bytes_80000000 = 1, + per_16bytes_8000 = 2, + per_16bytes_80 = 3, + per_16bytes_8 = 4, + per_element_ff = 5, + per_element_80000000 = 6, + per_element_8000 = 7, + per_element_80 = 8, + per_element_8 = 9, + per_16bytes_80000000_paired = 10, +}; + +/// NVVM vote sync kind. +pub const VoteSyncKind = enum(u32) { + any = 0, + all = 1, + ballot = 2, + uni = 3, +}; + +/// WGMMA overflow options. +pub const WGMMAScaleIn = enum(u32) { + one = 1, + neg = 2, +}; + +/// WGMMA input predicate. +pub const WGMMAScaleOut = enum(u32) { + zero = 0, + one = 1, +}; + +/// NVVM WGMMA types. +pub const WGMMATypes = enum(u32) { + f16 = 0, + tf32 = 1, + u8 = 2, + s8 = 3, + b1 = 4, + bf16 = 5, + e4m3 = 6, + e5m2 = 7, + f32 = 8, + s32 = 9, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#nvvm.atomic_op`: operations supported by atom instruction +pub const AtomicOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMAtomicOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atomic_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomicOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomicOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.reduction`: NVVM barrier reduction operation +pub const BarrierReductionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BarrierReduction { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +pub const BarrierReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "barrier_redux_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BarrierReduxKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.block_scale_format`: MMA Block Scale Format +pub const BlockScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBlockScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "block_scale_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BlockScaleFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BlockScaleFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cta_group`: NVVM CTA group kind +pub const CTAGroupKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCTAGroupKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cta_group"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CTAGroupKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CTAGroupKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +pub const CVTPackFloatKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCVTPackFloatKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "packfloat_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CVTPackFloatKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CVTPackFloatKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +pub const CacheEvictionPriorityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCacheEvictionPriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cache_eviction_priority"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CacheEvictionPriority, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +pub const ClusterLaunchControlQueryTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMClusterLaunchControlQueryType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cluster_launch_control_query_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ClusterLaunchControlQueryType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ClusterLaunchControlQueryType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.op`: Comparison operator encoding +pub const CompareOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCompareOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CompareOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CompareOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +pub const ConvertFP4TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP4Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_fp4_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP4Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertFP4Type { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +pub const ConvertFP8TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP8Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_fp8_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP8Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertFP8Type { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +pub const ConvertScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_scale_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertScaleKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +pub const DotAccumulateTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMDotAccumulateType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dot_accumulate_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DotAccumulateType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) DotAccumulateType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +pub const EvictKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMEvictKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "evict_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: EvictKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) EvictKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +pub const FPRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMFPRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "fp_rnd_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FPRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) FPRoundingMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.grid_dep_action`: Action kind for grid dependency control +pub const GridDepActionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMGridDepActionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "grid_dep_action"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GridDepActionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) GridDepActionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +pub const IntegerRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMIntegerRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "int_rnd_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IntegerRoundingMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +pub const L2PrefetchSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVML2PrefetchSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "l2_prefetch"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: L2PrefetchSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) L2PrefetchSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +pub const LdStMatrixEltTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixEltType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_elt_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdStMatrixEltType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdStMatrixEltType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +pub const LoadCacheModifierExtKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierExtKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier_ext"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierExtKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheModifierExtKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +pub const LoadCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheModifierKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_shape`: shape attribute for ldmatrix +pub const LoadShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_src_format`: source format for ldmatrix +pub const LoadSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_src_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadSrcFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +pub const MBarrierLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +pub const MBarrierPhaseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierPhase; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_phase"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierPhase, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierPhase { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +pub const MBarrierScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_scope"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_space`: NVVM MBarrier space kind +pub const MBarrierSpaceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierSpaceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierSpaceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierSpaceKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +pub const MBarrierTxnKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierTxnKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_txn_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierTxnKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierTxnKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +pub const MBarrierWaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierWaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_wait"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierWaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierWaitKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_b1op`: MMA binary operations +pub const MMAB1OpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAB1Op; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_b1op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAB1Op, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAB1Op { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.block_scale_kind`: Block Scale Kind +pub const MMABlockScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMABlockScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "block_scale_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMABlockScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMABlockScaleKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_cta_count`: MMA CTA count +pub const MMACtaCountAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMACtaCount; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_cta_count"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMACtaCount, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMACtaCount { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_frag`: NVVM MMA frag type +pub const MMAFragAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAFrag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_frag"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAFrag, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAFrag { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_int_overflow`: MMA overflow options +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAIntOverflow { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_kind`: MMA operation kind +pub const MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_layout`: NVVM MMA layout +pub const MMALayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMALayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMALayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMALayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_type`: NVVM MMA types +pub const MMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMATypes { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.match_sync_kind`: NVVM match sync kind +pub const MatchSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMatchSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "match_sync_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MatchSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MatchSyncKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mem_order`: NVVM Memory Ordering kind +pub const MemOrderKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemOrderKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mem_order"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemOrderKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mem_scope`: NVVM Memory Scope kind +pub const MemScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mem_scope"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mul_mode`: multiply mode attribute +pub const MulModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMulMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mul_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MulMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MulMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.memory_space`: NVVM Memory Space +pub const NVVMMemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMNVVMMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: NVVMMemorySpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) NVVMMemorySpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.permute_mode`: NVVM permute mode +pub const PermuteModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPermuteMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "permute_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PermuteMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PermuteMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +pub const PrefetchCacheLevelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPrefetchCacheLevel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "prefetch_cache_level"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PrefetchCacheLevel, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PrefetchCacheLevel { + return @fromBackingInt(@intCast(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.proxy_kind`: Proxy kind +pub const ProxyKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMProxyKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "proxy_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ProxyKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ProxyKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.red_op`: Ops supported by red instruction +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "red_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionOp { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.red_type`: types supported by red instruction +pub const ReductionTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "red_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +pub const SPCompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_elem_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressElemSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +pub const SPCompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressFactorType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +pub const SPCompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_index_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressIndexSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_op_kind`: spcompress operation kind +pub const SPCompressOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_op_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +pub const SPCompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_rep_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressRepFactor { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +pub const SPDecompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_elem_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressElemSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +pub const SPDecompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressFactorType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +pub const SPDecompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_index_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressIndexSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +pub const SPDecompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_rep_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressRepFactor { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sat_mode`: NVVM SaturationMode kind +pub const SaturationModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sat_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SaturationMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sat`: NVVM SaturationMode kind +pub const SaturationModeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationModeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sat"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationModeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SaturationModeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +pub const ScaleVecSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMScaleVecSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "scale_vec_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ScaleVecSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ScaleVecSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.action`: NVVM set max register action +pub const SetMaxRegisterActionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSetMaxRegisterAction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "action"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SetMaxRegisterAction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SetMaxRegisterAction { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.shared_space`: Shared memory space +pub const SharedSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSharedSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shared_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SharedSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.shfl_kind`: NVVM shuffle kind +pub const ShflKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMShflKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shfl_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShflKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ShflKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sparsity_format`: MMA Sparsity Format +pub const SparsityFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSparsityFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparsity_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SparsityFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SparsityFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.state_space`: NVVM State Space +pub const StateSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStateSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "state_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StateSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StateSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +pub const StoreCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "store_cache_modifier"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StoreCacheModifierKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.store_shape`: shape attribute for stmatrix +pub const StoreShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "store_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StoreShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +pub const TCBarParamAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTCBarParam; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "TCBarParam"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TCBarParam, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TCBarParam { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +pub const TMALoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMALoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMALoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMALoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +pub const TMAReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_redux_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMAReduxKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +pub const TMAStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMAStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +pub const Tcgen05CpMulticastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpMulticast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_multicast"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpMulticast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpMulticast { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +pub const Tcgen05CpShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +pub const Tcgen05CpSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_src_fmt"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpSrcFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +pub const Tcgen05FenceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05FenceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_fence"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05FenceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05FenceKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 +pub const Tcgen05LdStShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05LdStShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_ldst_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05LdStShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05LdStShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +pub const Tcgen05MMABlockScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMABlockScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_block_scale"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMABlockScale, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMABlockScale { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +pub const Tcgen05MMACollectorBBufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorb"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorBBuffer, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMACollectorBBuffer { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +pub const Tcgen05MMACollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorop"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMACollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +pub const Tcgen05MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMAKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +pub const Tcgen05WaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05WaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_wait"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05WaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05WaitKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +pub const TensormapElemtypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapElemtype; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_elemtype"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapElemtype, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapElemtype { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +pub const TensormapFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_field"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapField, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapField { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +pub const TensormapFillModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapFillMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_fill_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapFillMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapFillMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +pub const TensormapInterleaveLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapInterleaveLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_interleave_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapInterleaveLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapInterleaveLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +pub const TensormapSwizzleAtomicityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_atomicity"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleAtomicity, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapSwizzleAtomicity { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +pub const TensormapSwizzleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapSwizzleMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +pub const TmemLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTmemLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "TmemLayout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.validate_pattern`: NVVM validate data pattern +pub const ValidatePatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMValidatePattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "validate_pattern"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ValidatePattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ValidatePattern { + return @fromBackingInt(@intCast(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.vote_sync_kind`: NVVM vote sync kind +pub const VoteSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMVoteSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "vote_sync_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: VoteSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) VoteSyncKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_scale_in`: WGMMA overflow options +pub const WGMMAScaleInAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleIn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_in"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleIn, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMAScaleIn { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_scale_out`: WGMMA input predicate +pub const WGMMAScaleOutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleOut; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_out"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleOut, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMAScaleOut { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_type`: NVVM WGMMA types +pub const WGMMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMATypes { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +pub const LdStMatrixShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixShapeAttrGet(ctx.ptr(), args.m, args.n); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn m(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetM(self.ptr()); + } + pub fn n(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetN(self.ptr()); + } +}; + +/// `#nvvm.shape`: Shape of an MMA operation +pub const MMAShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + k: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAShapeAttrGet(ctx.ptr(), args.m, args.n, args.k); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn m(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetM(self.ptr()); + } + pub fn n(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetN(self.ptr()); + } + pub fn k(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetK(self.ptr()); + } +}; + +/// `#nvvm.target`: GPU target of an NVVM module +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "target"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + O: c_int = 2, + triple: []const u8 = "nvptx64-nvidia-cuda", + chip: []const u8 = "sm_75", + features: []const u8 = "", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + verifyTarget: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTargetAttrGet(ctx.ptr(), args.O, mlir.stringRef(args.triple), mlir.stringRef(args.chip), mlir.stringRef(args.features), if (args.flags) |v| v.ptr() else .{ .ptr = null }, if (args.link) |v| v.ptr() else .{ .ptr = null }, args.verifyTarget); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn o(self: *const Self) c_int { + return c.mlirCuteNVVMTargetAttrGetO(self.ptr()); + } + pub fn triple(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetTriple(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn chip(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetChip(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn features(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetFeatures(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn flags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetFlags(self.ptr()).ptr); + } + pub fn link(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetLink(self.ptr()).ptr); + } + pub fn verifyTarget(self: *const Self) bool { + return c.mlirCuteNVVMTargetAttrGetVerifyTarget(self.ptr()); + } +}; + +// Operation builders, generated from the operations' .td. +/// `nvvm.add.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.addf`. Result types are explicit; attributes use mlir.Attribute. +pub fn addf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.addf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk( + ctx: *mlir.Context, + ptr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk", .{ + .operands = .{ .flat = &.{ ptr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + coordinates, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.atomicrmw`. Result types are explicit; attributes use mlir.Attribute. +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + a: *const mlir.Value, + res_type: *const mlir.Type, + op: *const mlir.Attribute, + opts: struct { + b: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + syncscope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); + return mlir.Operation.make(ctx, "nvvm.atomicrmw", .{ + .operands = .{ .flat = if (opts.b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.bar.warp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ + .operands = .{ .flat = &.{mask} }, + .location = location, + }); +} + +/// `nvvm.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier( + ctx: *mlir.Context, + opts: struct { + barrierId: ?*const mlir.Value = null, + numberOfThreads: ?*const mlir.Value = null, + reductionPredicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + reductionOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); + return mlir.Operation.make(ctx, "nvvm.barrier", .{ + .operands = .{ .variadic = &.{ + if (opts.barrierId) |value| &.{value} else &.{}, + if (opts.numberOfThreads) |value| &.{value} else &.{}, + if (opts.reductionPredicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_arrive( + ctx: *mlir.Context, + numberOfThreads: *const mlir.Value, + opts: struct { + barrierId: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier.arrive", .{ + .operands = .{ .flat = if (opts.barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, + .location = location, + }); +} + +/// `nvvm.barrier.cta.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_arrive( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + threadCount: *const mlir.Value, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.arrive", .{ + .operands = .{ .flat = &.{ barrierId, threadCount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_red( + ctx: *mlir.Context, + pred: *const mlir.Value, + barrierId: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.red", .{ + .operands = .{ .flat = if (opts.threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_sync( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.sync", .{ + .operands = .{ .flat = if (opts.threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier0`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier0(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier0", .{ + .location = location, + }); +} + +/// `nvvm.breakpoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn breakpoint(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.breakpoint", .{ + .location = location, + }); +} + +/// `nvvm.clmad`. Result types are explicit; attributes use mlir.Attribute. +pub fn clmad( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + high: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); + return mlir.Operation.make(ctx, "nvvm.clmad", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive.relaxed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive_relaxed( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive.relaxed", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_wait( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.query.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_query_cancel(ctx: *mlir.Context, try_cancel_response: *const mlir.Value, res_type: *const mlir.Type, query_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.query.cancel", .{ + .operands = .{ .flat = &.{try_cancel_response} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "query_type", query_type), + }, + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.try.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_try_cancel( + ctx: *mlir.Context, + smemAddress: *const mlir.Value, + mbarrier: *const mlir.Value, + opts: struct { + multicast: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.try.cancel", .{ + .operands = .{ .flat = &.{ smemAddress, mbarrier } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.compare.and.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn compare_and_set(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.compare.and.set", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "op", op), + }, + .location = location, + }); +} + +/// `nvvm.convert.and.pack.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_and_pack_integer( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + dst_type: *const mlir.Type, + convert_type: *const mlir.Attribute, + opts: struct { + srcC: ?*const mlir.Value = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + attributes.appendAssumeCapacity(.named(ctx, "convert_type", convert_type)); + return mlir.Operation.make(ctx, "nvvm.convert.and.pack.integer", .{ + .operands = .{ .flat = if (opts.srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f8x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_s2f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.s2f6x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, value } else &.{a} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_bf16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f16x2", .{ + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f4x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstType: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_s2f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.s2f6x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f4x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f4x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f4x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f6x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f6x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f6x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f8x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f8x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f8x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f4x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f4x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f4x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f6x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f6x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f6x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_integer( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.tf32`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_tf32( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.tf32", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.s2f6x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_s2f6x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.s2f6x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cos`. Result types are explicit; attributes use mlir.Attribute. +pub fn cos( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.cos", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_global_shared_cta( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + byteMask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{size}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.byteMask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_prefetch( + ctx: *mlir.Context, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.prefetch", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + mbar: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{mbar}, + &.{size}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.shared.cta", .{ + .operands = .{ .flat = &.{ dstMem, srcMem, mbar, size } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + coordinates, + im2colOffsets, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + im2colOffsets, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + mbar: *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + isCTAOnly: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDescriptor}, + coordinates, + &.{mbar}, + im2colOffsets, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global_override( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDesc: *const mlir.Value, + mbar: *const mlir.Value, + overrideAddr: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + group: ?*const mlir.Attribute = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDesc}, + &.{mbar}, + &.{overrideAddr}, + coordinates, + im2colArgs, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.wait_group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_wait_group( + ctx: *mlir.Context, + group: *const mlir.Attribute, + opts: struct { + read: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "group", group)); + if (opts.read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + noinc: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.mbarrier.arrive", .{ + .operands = .{ .flat = &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.shared.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_shared_global( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + size: *const mlir.Attribute, + modifier: *const mlir.Attribute, + opts: struct { + cpSize: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.shared.global", .{ + .operands = .{ .flat = if (opts.cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, + .attributes = &.{ + .named(ctx, "size", size), + .named(ctx, "modifier", modifier), + }, + .location = location, + }); +} + +/// `nvvm.cp.async.wait.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_wait_group(ctx: *mlir.Context, n: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ + .attributes = &.{ + .named(ctx, "n", n), + }, + .location = location, + }); +} + +/// `nvvm.cvt.packfloat`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + from: *const mlir.Attribute, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "from", from)); + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat", .{ + .operands = .{ .flat = &.{ srcA, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.packfloat.f32`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.to.f4x8.packed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_to_f4x8_packed( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + dstType: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.cvt.to.f4x8.packed", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.dot.accumulate.2way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_2way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, b_hi: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.2way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "b_hi", b_hi), + }, + .location = location, + }); +} + +/// `nvvm.dot.accumulate.4way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_4way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.4way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + }, + .location = location, + }); +} + +/// `nvvm.elect.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn elect_sync( + ctx: *mlir.Context, + pred_type: *const mlir.Type, + opts: struct { + membermask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ + .operands = .{ .flat = if (opts.membermask) |value| &.{value} else &.{} }, + .results = .{ .flat = &.{pred_type} }, + .location = location, + }); +} + +/// `nvvm.ex2`. Result types are explicit; attributes use mlir.Attribute. +pub fn ex2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.ex2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.exit`. Result types are explicit; attributes use mlir.Attribute. +pub fn exit(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.exit", .{ + .location = location, + }); +} + +/// `nvvm.fabs`. Result types are explicit; attributes use mlir.Attribute. +pub fn fabs( + ctx: *mlir.Context, + value_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fabs", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ + .location = location, + }); +} + +/// `nvvm.fence.proxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy( + ctx: *mlir.Context, + kind: *const mlir.Attribute, + opts: struct { + space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.acquire`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_acquire( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.acquire", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.release`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_release( + ctx: *mlir.Context, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.release", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_sync_restrict( + ctx: *mlir.Context, + order: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "order", order)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.sync_restrict", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.sc.cluster`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sc_cluster(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sc.cluster", .{ + .location = location, + }); +} + +/// `nvvm.fence.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sync_restrict", .{ + .attributes = &.{ + .named(ctx, "order", order), + }, + .location = location, + }); +} + +/// `nvvm.fma`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + rnd: *const mlir.Attribute, + opts: struct { + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + oob: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "rnd", rnd)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); + return mlir.Operation.make(ctx, "nvvm.fma", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_f16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmax`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmax( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmax", .{ + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmin`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmin( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmin", .{ + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.griddepcontrol`. Result types are explicit; attributes use mlir.Attribute. +pub fn griddepcontrol(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.griddepcontrol", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.inline_ptx`. Result types are explicit; attributes use mlir.Attribute. +pub fn inline_ptx( + ctx: *mlir.Context, + readOnlyArgs: []const *const mlir.Value, + readWriteArgs: []const *const mlir.Value, + ptxCode: *const mlir.Attribute, + opts: struct { + predicate: ?*const mlir.Value = null, + writeOnlyArgs_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "nvvm.inline_ptx", .{ + .operands = .{ .variadic = &.{ + readOnlyArgs, + readWriteArgs, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = &.{ + .named(ctx, "ptxCode", ptxCode), + }, + .location = location, + }); +} + +/// `nvvm.ldmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn ldmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + num: *const mlir.Attribute, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num", num)); + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.ldmatrix", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.load.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_ext( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + prefetch_: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + unified: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + if (opts.unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); + return mlir.Operation.make(ctx, "nvvm.load.ext", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.log2`. Result types are explicit; attributes use mlir.Attribute. +pub fn log2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.log2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mapa`. Result types are explicit; attributes use mlir.Attribute. +pub fn mapa(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mapa", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.match.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn match_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.match.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.expect_tx", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.check.layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_check_layout(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.check.layout", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.complete_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_complete_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + report: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.complete_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (opts.report) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.expect_tx", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_init( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.inval`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_inval(ctx: *mlir.Context, addr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.inval", .{ + .operands = .{ .flat = &.{addr} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.test.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_test_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.test.wait", .{ + .operands = .{ .flat = &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ticks: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait", .{ + .operands = .{ .flat = if (opts.ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + ticks: *const mlir.Value, + opts: struct { + useIntrinsic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase, ticks } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity.timelimit", .{ + .operands = .{ .flat = &.{ addr, phase, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.timelimit", .{ + .operands = .{ .flat = &.{ addr, state, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + multicast: ?*const mlir.Value = null, + space: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn", .{ + .operands = .{ .flat = if (opts.multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn_cta( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn.cta", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait", .{ + .operands = .{ .flat = &.{ addr, state } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.memory.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_barrier(ctx: *mlir.Context, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.memory.barrier", .{ + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + opts: struct { + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + sp_metadata: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + sparsityFormat: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 11) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "sparsityFormat", sparsityFormat)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC, sp_metadata, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_smem_desc( + ctx: *mlir.Context, + pointer: *const mlir.Value, + ldm: *const mlir.Value, + stride: *const mlir.Value, + baseOffset: *const mlir.Value, + swizzle: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + mmaDescVersion: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); + return mlir.Operation.make(ctx, "nvvm.mma_smem_desc", .{ + .operands = .{ .flat = &.{ pointer, ldm, stride, baseOffset, swizzle } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.movmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn movmatrix( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + shape: *const mlir.Attribute, + eltType: *const mlir.Attribute, + opts: struct { + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + attributes.appendAssumeCapacity(.named(ctx, "eltType", eltType)); + return mlir.Operation.make(ctx, "nvvm.movmatrix", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.mul", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.bf16x2.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_bf16x2_f16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.bf16x2.f16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f16x2.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f16x2_bf16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f16x2.bf16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.mul.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.ld.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_ld_reduce( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + ordering: *const mlir.Attribute, + op: *const mlir.Attribute, + dataType: *const mlir.Attribute, + opts: struct { + accPrec: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "ordering", ordering)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "dataType", dataType)); + if (opts.accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); + return mlir.Operation.make(ctx, "nvvm.multimem.ld.reduce", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_red(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.red", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "op", op), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.multimem.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_st(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.st", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.nanosleep`. Result types are explicit; attributes use mlir.Attribute. +pub fn nanosleep(ctx: *mlir.Context, duration: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.nanosleep", .{ + .operands = .{ .flat = &.{duration} }, + .location = location, + }); +} + +/// `nvvm.pmevent`. Result types are explicit; attributes use mlir.Attribute. +pub fn pmevent( + ctx: *mlir.Context, + opts: struct { + maskedEventId: ?*const mlir.Attribute = null, + eventId: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); + if (opts.eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); + return mlir.Operation.make(ctx, "nvvm.pmevent", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + cacheLevel: ?*const mlir.Attribute = null, + evictPriority: ?*const mlir.Attribute = null, + tensormap: ?*const mlir.Attribute = null, + uniform: ?*const mlir.Attribute = null, + in_param_space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); + if (opts.evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); + if (opts.tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); + if (opts.uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); + if (opts.in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); + return mlir.Operation.make(ctx, "nvvm.prefetch", .{ + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prmt`. Result types are explicit; attributes use mlir.Attribute. +pub fn prmt( + ctx: *mlir.Context, + lo: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + hi: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.prmt", .{ + .operands = .{ .flat = if (opts.hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `nvvm.rcp.approx.ftz.f`. Result types are explicit; attributes use mlir.Attribute. +pub fn rcp_approx_ftz_f(ctx: *mlir.Context, arg: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.aggr.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_aggr_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.aggr.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock64`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock64(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock64", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.dynamic.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_dynamic_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.dynamic.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg0`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg0(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg0", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg1`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg1(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg1", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg10`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg10(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg10", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg11`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg11(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg11", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg12`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg12(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg12", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg13`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg13(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg13", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg14`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg14(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg14", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg15`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg15(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg15", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg16`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg16(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg16", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg17`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg17(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg17", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg18`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg18(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg18", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg19`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg19(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg19", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg2`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg2(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg2", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg20`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg20(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg20", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg21`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg21(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg21", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg22`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg22(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg22", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg23`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg23(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg23", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg24`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg24(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg24", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg25`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg25(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg25", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg26`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg26(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg26", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg27`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg27(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg27", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg28`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg28(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg28", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg29`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg29(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg29", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg3`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg3(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg3", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg30`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg30(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg30", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg31`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg31(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg31", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg4`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg4(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg4", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg5`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg5(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg5", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg6`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg6(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg6", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg7`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg7(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg7", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg8`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg8(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg8", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg9`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg9(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg9", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer.lo`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer_lo(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer.lo", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.gridid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_gridid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.gridid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.laneid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_laneid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.laneid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.eq`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_eq(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.eq", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.ge`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_ge(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.ge", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.gt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_gt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.gt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.le`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_le(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.le", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.lt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_lt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.lt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nsmid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nsmid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nsmid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nwarpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nwarpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nwarpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.smid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_smid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.smid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.total.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_total_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.total.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn red( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + op: *const mlir.Attribute, + type_: *const mlir.Attribute, + opts: struct { + cacheHint: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + memScope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "type", type_)); + return mlir.Operation.make(ctx, "nvvm.red", .{ + .operands = .{ .flat = if (opts.cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.redux.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn redux_sync( + ctx: *mlir.Context, + val: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + return mlir.Operation.make(ctx, "nvvm.redux.sync", .{ + .operands = .{ .flat = &.{ val, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.rsqrt`. Result types are explicit; attributes use mlir.Attribute. +pub fn rsqrt( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.rsqrt", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.setmaxregister`. Result types are explicit; attributes use mlir.Attribute. +pub fn setmaxregister(ctx: *mlir.Context, regCount: *const mlir.Attribute, action: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.setmaxregister", .{ + .attributes = &.{ + .named(ctx, "regCount", regCount), + .named(ctx, "action", action), + }, + .location = location, + }); +} + +/// `nvvm.shfl.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn shfl_sync( + ctx: *mlir.Context, + thread_mask: *const mlir.Value, + val: *const mlir.Value, + offset: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + return_value_and_is_valid: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); + return mlir.Operation.make(ctx, "nvvm.shfl.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val, offset, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sin`. Result types are explicit; attributes use mlir.Attribute. +pub fn sin( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sin", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spcompress(ctx: *mlir.Context, spDesc: *const mlir.Value, data: *const mlir.Value, metadata_type: *const mlir.Type, compressed_data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spcompress", .{ + .operands = .{ .flat = &.{ spDesc, data } }, + .results = .{ .flat = &.{ metadata_type, compressed_data_type } }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.spdecompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spdecompress(ctx: *mlir.Context, metadata: *const mlir.Value, compressed_data: *const mlir.Value, data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spdecompress", .{ + .operands = .{ .flat = &.{ metadata, compressed_data } }, + .results = .{ .flat = &.{data_type} }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.st.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn st_bulk( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + initVal: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); + return mlir.Operation.make(ctx, "nvvm.st.bulk", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.stmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn stmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + sources: []const *const mlir.Value, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(sources); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + return mlir.Operation.make(ctx, "nvvm.stmatrix", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.store.async`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_async(ctx: *mlir.Context, addr: *const mlir.Value, value_: *const mlir.Value, mbarrier: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.store.async", .{ + .operands = .{ .flat = &.{ addr, value_, mbarrier } }, + .location = location, + }); +} + +/// `nvvm.store.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_ext( + ctx: *mlir.Context, + value_: *const mlir.Value, + addr: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + return mlir.Operation.make(ctx, "nvvm.store.ext", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.subf`. Result types are explicit; attributes use mlir.Attribute. +pub fn subf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.subf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_alloc( + ctx: *mlir.Context, + addr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ + .operands = .{ .flat = &.{ addr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.commit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_commit( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + smem_a_read: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.cp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_cp( + ctx: *mlir.Context, + taddr: *const mlir.Value, + smem_desc: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + group: ?*const mlir.Attribute = null, + multicast: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.cp", .{ + .operands = .{ .flat = &.{ taddr, smem_desc } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.dealloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_dealloc( + ctx: *mlir.Context, + taddr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ + .operands = .{ .flat = &.{ taddr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.fence`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05.ld`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + pack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red( + ctx: *mlir.Context, + addr: *const mlir.Value, + data_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + op: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{ data_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale.decompress_b", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + disableOutputLane: ?*const mlir.Value = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.decompress_b", .{ + .operands = .{ .flat = if (opts.disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + &.{sparseMetadata}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws", .{ + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws.sp", .{ + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc(ctx: *mlir.Context, startAddr: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, swizzleMode: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma_smem_desc", .{ + .operands = .{ .flat = &.{ startAddr, leadingDimOffset, strideDimOffset, baseOffset, leadingDimMode, swizzleMode } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tcgen05.relinquish_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_relinquish_alloc_permit( + ctx: *mlir.Context, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.relinquish_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.shift`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_shift( + ctx: *mlir.Context, + taddr: *const mlir.Value, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.shift", .{ + .operands = .{ .flat = &.{taddr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_st( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + val: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + unpack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05_mma_smem_desc_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc_v2(ctx: *mlir.Context, startAddress: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, descriptorVersion: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, kSegmentOffset: *const mlir.Value, swizzleType: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05_mma_smem_desc_v2", .{ + .operands = .{ .flat = &.{ startAddress, leadingDimOffset, strideDimOffset, descriptorVersion, baseOffset, leadingDimMode, kSegmentOffset, swizzleType } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tensormap.cp_fenceproxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_cp_fenceproxy(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tensormap.cp_fenceproxy", .{ + .operands = .{ .flat = &.{ dst, src, size } }, + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.tensormap.replace`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_replace( + ctx: *mlir.Context, + addr: *const mlir.Value, + field: *const mlir.Attribute, + opts: struct { + new_value: ?*const mlir.Value = null, + ord: ?*const mlir.Attribute = null, + new_value_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "field", field)); + if (opts.ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); + if (opts.new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); + return mlir.Operation.make(ctx, "nvvm.tensormap.replace", .{ + .operands = .{ .flat = if (opts.new_value) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.trace.mark`. Result types are explicit; attributes use mlir.Attribute. +pub fn trace_mark( + ctx: *mlir.Context, + eventType: *const mlir.Attribute, + domain: *const mlir.Attribute, + event: *const mlir.Attribute, + opts: struct { + payload: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + payloadDescriptor: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "eventType", eventType)); + attributes.appendAssumeCapacity(.named(ctx, "domain", domain)); + attributes.appendAssumeCapacity(.named(ctx, "event", event)); + if (opts.payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); + return mlir.Operation.make(ctx, "nvvm.trace.mark", .{ + .operands = .{ .variadic = &.{ + if (opts.payload) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.vote.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn vote_sync(ctx: *mlir.Context, mask: *const mlir.Value, pred: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.vote.sync", .{ + .operands = .{ .flat = &.{ mask, pred } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.wgmma.commit.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_commit_group_sync_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.commit.group.sync.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.fence.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_fence_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.fence.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.mma_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_mma_async( + ctx: *mlir.Context, + inouts: *const mlir.Value, + descriptorA: *const mlir.Value, + descriptorB: *const mlir.Value, + results__type: *const mlir.Type, + shape: *const mlir.Attribute, + typeA: *const mlir.Attribute, + typeB: *const mlir.Attribute, + typeD: *const mlir.Attribute, + scaleD: *const mlir.Attribute, + scaleA: *const mlir.Attribute, + scaleB: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + satfinite: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "typeA", typeA)); + attributes.appendAssumeCapacity(.named(ctx, "typeB", typeB)); + attributes.appendAssumeCapacity(.named(ctx, "typeD", typeD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleD", scaleD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleA", scaleA)); + attributes.appendAssumeCapacity(.named(ctx, "scaleB", scaleB)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (opts.satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); + return mlir.Operation.make(ctx, "nvvm.wgmma.mma_async", .{ + .operands = .{ .flat = &.{ inouts, descriptorA, descriptorB } }, + .results = .{ .flat = &.{results__type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.wgmma.wait.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_wait_group_sync_aligned(ctx: *mlir.Context, group: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.wait.group.sync.aligned", .{ + .attributes = &.{ + .named(ctx, "group", group), + }, + .location = location, + }); +} + +/// `nvvm.wmma.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_load(ctx: *mlir.Context, ptr: *const mlir.Value, stride: *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, frag: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.load", .{ + .operands = .{ .flat = &.{ ptr, stride } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + .named(ctx, "frag", frag), + }, + .location = location, + }); +} + +/// `nvvm.wmma.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_mma(ctx: *mlir.Context, args: []const *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, eltypeA: *const mlir.Attribute, eltypeB: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.mma", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layoutA", layoutA), + .named(ctx, "layoutB", layoutB), + .named(ctx, "eltypeA", eltypeA), + .named(ctx, "eltypeB", eltypeB), + }, + .location = location, + }); +} + +/// `nvvm.wmma.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_store(ctx: *mlir.Context, ptr: *const mlir.Value, args: []const *const mlir.Value, stride: *const mlir.Value, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(args); + operands.appendAssumeCapacity(stride); + return mlir.Operation.make(ctx, "nvvm.wmma.store", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + }, + .location = location, + }); +} diff --git a/mlir/dialects/nvvm/test.zig b/mlir/dialects/nvvm/test.zig new file mode 100644 index 00000000..b908f6f4 --- /dev/null +++ b/mlir/dialects/nvvm/test.zig @@ -0,0 +1,481 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const operation_names: []const []const u8 = &.{ + "nvvm.add.packed.bf16x2.f32x2.f32x2", + "nvvm.add.packed.f16x2.f32x2.f32x2", + "nvvm.add.packed.f32x2", + "nvvm.add.packed.f32x2.bf16x2.f32x2", + "nvvm.add.packed.f32x2.f16x2.f32x2", + "nvvm.addf", + "nvvm.applypriority.async.bulk", + "nvvm.applypriority.async.bulk.tensor", + "nvvm.applypriority.async.bulk.tensor.override", + "nvvm.atomicrmw", + "nvvm.bar.warp.sync", + "nvvm.barrier", + "nvvm.barrier.arrive", + "nvvm.barrier.cta.arrive", + "nvvm.barrier.cta.red", + "nvvm.barrier.cta.sync", + "nvvm.barrier0", + "nvvm.breakpoint", + "nvvm.clmad", + "nvvm.cluster.arrive", + "nvvm.cluster.arrive.relaxed", + "nvvm.cluster.wait", + "nvvm.clusterlaunchcontrol.query.cancel", + "nvvm.clusterlaunchcontrol.try.cancel", + "nvvm.compare.and.set", + "nvvm.convert.and.pack.integer", + "nvvm.convert.bf16x2.to.f4x2", + "nvvm.convert.bf16x2.to.f6x2", + "nvvm.convert.bf16x2.to.f8x2", + "nvvm.convert.bf16x2.to.s2f6x2", + "nvvm.convert.f16x2.to.f4x2", + "nvvm.convert.f16x2.to.f6x2", + "nvvm.convert.f16x2.to.f8x2", + "nvvm.convert.f32x2.to.bf16x2", + "nvvm.convert.f32x2.to.f16x2", + "nvvm.convert.f32x2.to.f4x2", + "nvvm.convert.f32x2.to.f6x2", + "nvvm.convert.f32x2.to.f8x2", + "nvvm.convert.f32x2.to.s2f6x2", + "nvvm.convert.f32x4.to.f4x4", + "nvvm.convert.f32x4.to.f6x4", + "nvvm.convert.f32x4.to.f8x4", + "nvvm.convert.f4x2.to.f16x2", + "nvvm.convert.f6x2.to.f16x2", + "nvvm.convert.f8x2.to.bf16x2", + "nvvm.convert.f8x2.to.f16x2", + "nvvm.convert.float.to.integer", + "nvvm.convert.float.to.tf32", + "nvvm.convert.s2f6x2.to.bf16x2", + "nvvm.cos", + "nvvm.cp.async.bulk.commit.group", + "nvvm.cp.async.bulk.global.shared.cta", + "nvvm.cp.async.bulk.prefetch", + "nvvm.cp.async.bulk.shared.cluster.global", + "nvvm.cp.async.bulk.shared.cluster.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta.override", + "nvvm.cp.async.bulk.tensor.prefetch", + "nvvm.cp.async.bulk.tensor.prefetch.override", + "nvvm.cp.async.bulk.tensor.reduce", + "nvvm.cp.async.bulk.tensor.reduce.override", + "nvvm.cp.async.bulk.tensor.shared.cluster.global", + "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", + "nvvm.cp.async.bulk.wait_group", + "nvvm.cp.async.commit.group", + "nvvm.cp.async.mbarrier.arrive", + "nvvm.cp.async.shared.global", + "nvvm.cp.async.wait.group", + "nvvm.cvt.packfloat", + "nvvm.cvt.packfloat.f32", + "nvvm.cvt.to.f4x8.packed", + "nvvm.dot.accumulate.2way", + "nvvm.dot.accumulate.4way", + "nvvm.elect.sync", + "nvvm.ex2", + "nvvm.exit", + "nvvm.fabs", + "nvvm.fence.mbarrier.init", + "nvvm.fence.proxy", + "nvvm.fence.proxy.acquire", + "nvvm.fence.proxy.release", + "nvvm.fence.proxy.sync_restrict", + "nvvm.fence.sc.cluster", + "nvvm.fence.sync_restrict", + "nvvm.fma", + "nvvm.fma.packed.f32x2", + "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", + "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", + "nvvm.fmax", + "nvvm.fmin", + "nvvm.griddepcontrol", + "nvvm.inline_ptx", + "nvvm.ldmatrix", + "nvvm.load.ext", + "nvvm.log2", + "nvvm.mapa", + "nvvm.match.sync", + "nvvm.mbarrier.arrive", + "nvvm.mbarrier.arrive.expect_tx", + "nvvm.mbarrier.arrive.nocomplete", + "nvvm.mbarrier.arrive_drop", + "nvvm.mbarrier.arrive_drop.expect_tx", + "nvvm.mbarrier.arrive_drop.nocomplete", + "nvvm.mbarrier.check.layout", + "nvvm.mbarrier.complete_tx", + "nvvm.mbarrier.expect_tx", + "nvvm.mbarrier.init", + "nvvm.mbarrier.inval", + "nvvm.mbarrier.test.wait", + "nvvm.mbarrier.try_wait", + "nvvm.mbarrier.try_wait.parity", + "nvvm.mbarrier.try_wait.parity.timelimit", + "nvvm.mbarrier.try_wait.timelimit", + "nvvm.mbarrier.txn", + "nvvm.mbarrier.txn.cta", + "nvvm.mbarrier.wait", + "nvvm.mbarrier.wait.parity", + "nvvm.memory.barrier", + "nvvm.mma.block_scale", + "nvvm.mma.block_scale.internal", + "nvvm.mma.sp.block_scale", + "nvvm.mma.sp.block_scale.internal", + "nvvm.mma.sp.sync", + "nvvm.mma.sync", + "nvvm.mma_smem_desc", + "nvvm.movmatrix", + "nvvm.mul", + "nvvm.mul.packed.bf16x2.bf16x2.f16x2", + "nvvm.mul.packed.bf16x2.f32x2.f32x2", + "nvvm.mul.packed.f16x2.f16x2.bf16x2", + "nvvm.mul.packed.f16x2.f32x2.f32x2", + "nvvm.mul.packed.f32x2", + "nvvm.multimem.ld.reduce", + "nvvm.multimem.red", + "nvvm.multimem.st", + "nvvm.nanosleep", + "nvvm.pmevent", + "nvvm.prefetch", + "nvvm.prmt", + "nvvm.rcp.approx.ftz.f", + "nvvm.read.ptx.sreg.aggr.smem.size", + "nvvm.read.ptx.sreg.clock", + "nvvm.read.ptx.sreg.clock64", + "nvvm.read.ptx.sreg.cluster.ctaid.x", + "nvvm.read.ptx.sreg.cluster.ctaid.y", + "nvvm.read.ptx.sreg.cluster.ctaid.z", + "nvvm.read.ptx.sreg.cluster.ctarank", + "nvvm.read.ptx.sreg.cluster.nctaid.x", + "nvvm.read.ptx.sreg.cluster.nctaid.y", + "nvvm.read.ptx.sreg.cluster.nctaid.z", + "nvvm.read.ptx.sreg.cluster.nctarank", + "nvvm.read.ptx.sreg.clusterid.x", + "nvvm.read.ptx.sreg.clusterid.y", + "nvvm.read.ptx.sreg.clusterid.z", + "nvvm.read.ptx.sreg.ctaid.x", + "nvvm.read.ptx.sreg.ctaid.y", + "nvvm.read.ptx.sreg.ctaid.z", + "nvvm.read.ptx.sreg.dynamic.smem.size", + "nvvm.read.ptx.sreg.envreg0", + "nvvm.read.ptx.sreg.envreg1", + "nvvm.read.ptx.sreg.envreg10", + "nvvm.read.ptx.sreg.envreg11", + "nvvm.read.ptx.sreg.envreg12", + "nvvm.read.ptx.sreg.envreg13", + "nvvm.read.ptx.sreg.envreg14", + "nvvm.read.ptx.sreg.envreg15", + "nvvm.read.ptx.sreg.envreg16", + "nvvm.read.ptx.sreg.envreg17", + "nvvm.read.ptx.sreg.envreg18", + "nvvm.read.ptx.sreg.envreg19", + "nvvm.read.ptx.sreg.envreg2", + "nvvm.read.ptx.sreg.envreg20", + "nvvm.read.ptx.sreg.envreg21", + "nvvm.read.ptx.sreg.envreg22", + "nvvm.read.ptx.sreg.envreg23", + "nvvm.read.ptx.sreg.envreg24", + "nvvm.read.ptx.sreg.envreg25", + "nvvm.read.ptx.sreg.envreg26", + "nvvm.read.ptx.sreg.envreg27", + "nvvm.read.ptx.sreg.envreg28", + "nvvm.read.ptx.sreg.envreg29", + "nvvm.read.ptx.sreg.envreg3", + "nvvm.read.ptx.sreg.envreg30", + "nvvm.read.ptx.sreg.envreg31", + "nvvm.read.ptx.sreg.envreg4", + "nvvm.read.ptx.sreg.envreg5", + "nvvm.read.ptx.sreg.envreg6", + "nvvm.read.ptx.sreg.envreg7", + "nvvm.read.ptx.sreg.envreg8", + "nvvm.read.ptx.sreg.envreg9", + "nvvm.read.ptx.sreg.globaltimer", + "nvvm.read.ptx.sreg.globaltimer.lo", + "nvvm.read.ptx.sreg.gridid", + "nvvm.read.ptx.sreg.laneid", + "nvvm.read.ptx.sreg.lanemask.eq", + "nvvm.read.ptx.sreg.lanemask.ge", + "nvvm.read.ptx.sreg.lanemask.gt", + "nvvm.read.ptx.sreg.lanemask.le", + "nvvm.read.ptx.sreg.lanemask.lt", + "nvvm.read.ptx.sreg.nclusterid.x", + "nvvm.read.ptx.sreg.nclusterid.y", + "nvvm.read.ptx.sreg.nclusterid.z", + "nvvm.read.ptx.sreg.nctaid.x", + "nvvm.read.ptx.sreg.nctaid.y", + "nvvm.read.ptx.sreg.nctaid.z", + "nvvm.read.ptx.sreg.nsmid", + "nvvm.read.ptx.sreg.ntid.x", + "nvvm.read.ptx.sreg.ntid.y", + "nvvm.read.ptx.sreg.ntid.z", + "nvvm.read.ptx.sreg.nwarpid", + "nvvm.read.ptx.sreg.smid", + "nvvm.read.ptx.sreg.tid.x", + "nvvm.read.ptx.sreg.tid.y", + "nvvm.read.ptx.sreg.tid.z", + "nvvm.read.ptx.sreg.total.smem.size", + "nvvm.read.ptx.sreg.warpid", + "nvvm.read.ptx.sreg.warpsize", + "nvvm.red", + "nvvm.redux.sync", + "nvvm.rsqrt", + "nvvm.setmaxregister", + "nvvm.shfl.sync", + "nvvm.sin", + "nvvm.spcompress", + "nvvm.spdecompress", + "nvvm.st.bulk", + "nvvm.stmatrix", + "nvvm.store.async", + "nvvm.store.ext", + "nvvm.sub.packed.bf16x2.f32x2.f32x2", + "nvvm.sub.packed.f16x2.f32x2.f32x2", + "nvvm.sub.packed.f32x2", + "nvvm.sub.packed.f32x2.bf16x2.f32x2", + "nvvm.sub.packed.f32x2.f16x2.f32x2", + "nvvm.subf", + "nvvm.tcgen05.alloc", + "nvvm.tcgen05.commit", + "nvvm.tcgen05.cp", + "nvvm.tcgen05.dealloc", + "nvvm.tcgen05.fence", + "nvvm.tcgen05.ld", + "nvvm.tcgen05.ld.red", + "nvvm.tcgen05.ld.red.spcompress", + "nvvm.tcgen05.ld.spcompress", + "nvvm.tcgen05.mma", + "nvvm.tcgen05.mma.block_scale", + "nvvm.tcgen05.mma.block_scale.decompress_b", + "nvvm.tcgen05.mma.decompress_b", + "nvvm.tcgen05.mma.sp", + "nvvm.tcgen05.mma.sp.block_scale", + "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws.sp", + "nvvm.tcgen05.mma_smem_desc", + "nvvm.tcgen05.relinquish_alloc_permit", + "nvvm.tcgen05.shift", + "nvvm.tcgen05.st", + "nvvm.tcgen05.wait", + "nvvm.tcgen05_mma_smem_desc_v2", + "nvvm.tensormap.cp_fenceproxy", + "nvvm.tensormap.replace", + "nvvm.trace.mark", + "nvvm.vote.sync", + "nvvm.wgmma.commit.group.sync.aligned", + "nvvm.wgmma.fence.aligned", + "nvvm.wgmma.mma_async", + "nvvm.wgmma.wait.group.sync.aligned", + "nvvm.wmma.load", + "nvvm.wmma.mma", + "nvvm.wmma.store", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(nvvm); + std.testing.refAllDecls(nvvm.ops); + inline for (comptime std.meta.declarations(nvvm.ops)) |decl| { + const T = @field(nvvm.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every enum attribute prints as the compiler does" { + @setEvalBranchQuota(1_000_000); + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (.{ + .{ nvvm.ops.AtomicOpKindAttr, "#nvvm" }, + .{ nvvm.ops.BarrierReductionAttr, "#nvvm.reduction<{s}>" }, + .{ nvvm.ops.BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, + .{ nvvm.ops.BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, + .{ nvvm.ops.CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, + .{ nvvm.ops.CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, + .{ nvvm.ops.CacheEvictionPriorityAttr, "#nvvm" }, + .{ nvvm.ops.ClusterLaunchControlQueryTypeAttr, "#nvvm" }, + .{ nvvm.ops.CompareOpKindAttr, "#nvvm.op<{s}>" }, + .{ nvvm.ops.ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, + .{ nvvm.ops.ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, + .{ nvvm.ops.ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, + .{ nvvm.ops.DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, + .{ nvvm.ops.EvictKindAttr, "#nvvm.evict_kind<{s}>" }, + .{ nvvm.ops.FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, + .{ nvvm.ops.GridDepActionKindAttr, "#nvvm" }, + .{ nvvm.ops.IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, + .{ nvvm.ops.L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, + .{ nvvm.ops.LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, + .{ nvvm.ops.LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, + .{ nvvm.ops.LoadCacheModifierKindAttr, "#nvvm" }, + .{ nvvm.ops.LoadShapeAttr, "#nvvm.load_shape<{s}>" }, + .{ nvvm.ops.LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, + .{ nvvm.ops.MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, + .{ nvvm.ops.MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, + .{ nvvm.ops.MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, + .{ nvvm.ops.MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, + .{ nvvm.ops.MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, + .{ nvvm.ops.MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, + .{ nvvm.ops.MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, + .{ nvvm.ops.MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, + .{ nvvm.ops.MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, + .{ nvvm.ops.MMAFragAttr, "#nvvm.mma_frag<{s}>" }, + .{ nvvm.ops.MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, + .{ nvvm.ops.MMAKindAttr, "#nvvm.mma_kind<{s}>" }, + .{ nvvm.ops.MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, + .{ nvvm.ops.MMATypesAttr, "#nvvm.mma_type<{s}>" }, + .{ nvvm.ops.MatchSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, + .{ nvvm.ops.MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, + .{ nvvm.ops.MulModeAttr, "#nvvm" }, + .{ nvvm.ops.NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, + .{ nvvm.ops.PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, + .{ nvvm.ops.PrefetchCacheLevelAttr, "#nvvm" }, + .{ nvvm.ops.ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, + .{ nvvm.ops.ReductionKindAttr, "#nvvm" }, + .{ nvvm.ops.ReductionOpAttr, "#nvvm.red_op<{s}>" }, + .{ nvvm.ops.ReductionTypeAttr, "#nvvm.red_type<{s}>" }, + .{ nvvm.ops.SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, + .{ nvvm.ops.SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, + .{ nvvm.ops.SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, + .{ nvvm.ops.SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, + .{ nvvm.ops.SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, + .{ nvvm.ops.SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, + .{ nvvm.ops.SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, + .{ nvvm.ops.SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, + .{ nvvm.ops.SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, + .{ nvvm.ops.SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, + .{ nvvm.ops.SaturationModeKindAttr, "#nvvm.sat<{s}>" }, + .{ nvvm.ops.ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, + .{ nvvm.ops.SetMaxRegisterActionAttr, "#nvvm" }, + .{ nvvm.ops.SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, + .{ nvvm.ops.ShflKindAttr, "#nvvm" }, + .{ nvvm.ops.SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, + .{ nvvm.ops.StateSpaceAttr, "#nvvm.state_space<{s}>" }, + .{ nvvm.ops.StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, + .{ nvvm.ops.StoreShapeAttr, "#nvvm.store_shape<{s}>" }, + .{ nvvm.ops.TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, + .{ nvvm.ops.TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, + .{ nvvm.ops.TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, + .{ nvvm.ops.TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, + .{ nvvm.ops.Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, + .{ nvvm.ops.Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, + .{ nvvm.ops.Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, + .{ nvvm.ops.Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, + .{ nvvm.ops.Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, + .{ nvvm.ops.Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, + .{ nvvm.ops.Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, + .{ nvvm.ops.Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, + .{ nvvm.ops.TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, + .{ nvvm.ops.TensormapFieldAttr, "#nvvm" }, + .{ nvvm.ops.TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, + .{ nvvm.ops.TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, + .{ nvvm.ops.TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, + .{ nvvm.ops.TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, + .{ nvvm.ops.TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, + .{ nvvm.ops.ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, + .{ nvvm.ops.VoteSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, + .{ nvvm.ops.WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, + .{ nvvm.ops.WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, + }) |example| { + const T = example[0]; + const E = @FieldType(T.InitArgs, "value"); + inline for (@typeInfo(E).@"enum".field_names) |name| { + const attr = try T.get(ctx, .{ .value = @field(E, name) }); + try std.testing.expectEqual(@field(E, name), attr.value()); + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{attr}); + try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "nvvm types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ nvvm.ops.MMAShapeAttr, "#nvvm.shape" }, + .{ nvvm.ops.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } +} + +test "reqntid and minctasm are kernel attributes of the nvvm dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ nvvm.reqntid(ctx, .{ 128, 1, 1 }), nvvm.minctasm(ctx, 1) })}); + try std.testing.expectEqualStrings("{nvvm.minctasm = 1 : i32, nvvm.reqntid = array}", w.buffered()); +} + +test "nvvm target rejects what the compiler rejects" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .O = 4 })); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .chip = "" })); + const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .link = not_strings })); +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/rocdl/BUILD.bazel b/mlir/dialects/rocdl/BUILD.bazel new file mode 100644 index 00000000..ef0d7583 --- /dev/null +++ b/mlir/dialects/rocdl/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPIROCDL", + "@llvm-project//mlir:ROCDLDialect", + ], +) + +zig_library( + name = "rocdl", + srcs = [ + "annotations.zig", + "attributes.zig", + "enums.zig", + "ops.zig", + "target.zig", + "types.zig", + ], + import_name = "mlir/dialects/rocdl", + main = "rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/llvm", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":capi", + ":rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/rocdl/README.md b/mlir/dialects/rocdl/README.md new file mode 100644 index 00000000..500e4657 --- /dev/null +++ b/mlir/dialects/rocdl/README.md @@ -0,0 +1,54 @@ +# ROCDL + +Depend on `//mlir/dialects/rocdl` and import `mlir/dialects/rocdl` for LLVM's +AMDGPU intrinsic dialect. The package links LLVM's `ROCDLDialect` and +`CAPIROCDL`. `fly_rocdl` is a separate dialect and package. + +The pinned LLVM revision supplies 374 operations, 15 attributes, and 12 enums. + +- `ops` provides every operation builder. Names replace dots with underscores, + such as `ops.raw_buffer_load` and `ops.mfma_f32_16x16x4f32`. Arguments are + operands, explicit result types, required attributes, an `opts` struct when + there are optional attributes, and the location, preceded by the MLIR context. + Attribute names match ODS. Optional and default-valued attributes are fields + of `opts` with `null` defaults; pass `.{}` to omit them and let MLIR supply + its defaults. Builders verify the resulting operation. +- Enum types and attribute wrappers are available at the module root and in + `enums`/`attributes`. Attribute constructors return `error.InvalidMlir` for + invalid enum values or flag bits. Bit enums support `combine` and `contains`. +- `TargetAttr` constructs and reads `#rocdl.target`, including optimization + level, triple, chip, features, ABI, flags, and link files. Defaults match LLVM. + Invalid fields and attributes from another context are rejected. +- `annotations` supplies function, module, and memory annotations. Apply + function annotations to `llvm.func`, and `xnack`/`sramecc` to LLVM-compatible + modules. A missing module option preserves the target default; `false` + explicitly disables the option. +- `types` supplies every builtin vector type ROCDL's TableGen records use, + named or inline, and LLVM pointers in ROCDL address spaces. ROCDL defines no independent + MLIR types. +- `registerDialect` registers ROCDL; loading it also loads its LLVM dependency. + The existing `cvtPkF32Fp8` convenience builder remains available. + +```zig +const rocdl = @import("mlir/dialects/rocdl"); + +const policy = try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) }); +const load = rocdl.ops.raw_buffer_load( + ctx, resource, offset, scalar_offset, + mlir.Type.float(ctx, .f32), .{ .aux = policy.attribute() }, location, +); +``` + +The `:capi` target exposes attribute constructors, predicates, and getters +through `capi.h`; dialect registration comes from upstream +`mlir-c/Dialect/ROCDL.h`. Declarations and implementations are grouped by +introspection (a test helper), attributes, and target configuration. Getters require a handle +accepted by the matching predicate. Contexts own all returned attributes and +strings. There are no additional C operation wrappers: Zig builders use MLIR's +standard operation API. + +Tests compare the generated operation list with the loaded dialect's registered +operation count, check every operation name, round-trip every named enum value, +reject invalid enum/target inputs, and verify and reparse representative barrier, +scheduling, buffer, and matrix operations in an LLVM function. The tests require +no AMD GPU or ROCm runtime. diff --git a/mlir/dialects/rocdl/REGENERATING.md b/mlir/dialects/rocdl/REGENERATING.md new file mode 100644 index 00000000..526cb17b --- /dev/null +++ b/mlir/dialects/rocdl/REGENERATING.md @@ -0,0 +1,62 @@ +# Regenerating the ROCDL bindings + +These are instructions for an agent. They describe what the generated ROCDL +bindings must contain after an LLVM bump, where to derive them from, and how to +check the result. Scripts and intermediate files are not committed; only the +generated bindings are. + +## Sources + +Use the LLVM revision Bazel pins, and read TableGen's resolved records rather +than the `.td` text: multiclasses instantiate many ROCDL operations. + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm_project="$(bazel info output_base)/external/+llvm+llvm-project" +bazel-bin/external/+llvm+llvm-project/llvm/llvm-tblgen --dump-json \ + -I "$llvm_project/mlir/include" -I "$llvm_project/llvm/include" \ + "$llvm_project/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td" \ + -o /tmp/rocdl-records.json +``` + +Every `Op`, `AttrDef` and `EnumInfo` record whose dialect is `ROCDL_Dialect` +belongs to the bindings. + +## What each file must contain + +Names, builder parameters, `opts`, wrappers, C API names and comments follow +`mlir/dialects/README.md`. Specific to ROCDL: + +- `ops.zig`: one builder per operation, listed in `test.zig`'s + `operation_names`. Follow argument constraints through `Arg` wrappers. +- `enums.zig`: one enum per ROCDL enum, with its bit width and values. Unlike + the README rule, a case whose IR spelling is not an identifier or is a + keyword takes the C++ symbol (`volatile` → `volatile_op`). Bit enums are + non-exhaustive and provide `combine` and `contains`. +- `attributes.zig`: one `Attr` wrapper per enum attribute, and + `enum_attributes` listing them. +- `types.zig`: `AddressSpace`, `pointerType`, and a `v` constructor + for every vector type the records use, named or inline. +- `capi.h`/`capi.cc`: `mlirROCDLGetOperationCount` under `// Introspection` + and, per enum attribute, the predicate, the getter (null for out-of-range + values) and `GetValue` under `// Attributes`. The `// Target` sections are + maintained by hand: in the + pinned revision `ROCDLTargetAttr::getChecked` delegates to `Base::get`, so the + C wrapper verifies target inputs first. + +Each generated Zig file starts with the "Generated from LLVM ROCDL TableGen +records" header. `target.zig`, `annotations.zig` and `rocdl.zig` are maintained +by hand; update their aliases and helpers when LLVM's surface changes. If the +dialect gains variadic operands, regions, successors, its own types, an enum +without an attribute, or new target fields or discardable attributes, extend +the bindings and this file. + +## Checks + +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/rocdl:test //dsl/fly:test //zml:test`. Beyond the + README's tests, the operation list is compared to + `mlirROCDLGetOperationCount`, and attribute tests exercise every named enum + case, named bit groups and combined bitfields. Add fixtures when a change + introduces a new operand layout or attribute representation. +- A regeneration without an LLVM change is byte-identical. diff --git a/mlir/dialects/rocdl/annotations.zig b/mlir/dialects/rocdl/annotations.zig new file mode 100644 index 00000000..85128473 --- /dev/null +++ b/mlir/dialects/rocdl/annotations.zig @@ -0,0 +1,56 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// Discardable ROCDL attributes for an `llvm.func` operation. +pub const FunctionOpts = struct { + kernel: bool = false, + reqdWorkGroupSize: ?[3]i32 = null, + flatWorkGroupSize: ?[]const u8 = null, + maxFlatWorkGroupSize: ?i32 = null, + wavesPerEu: ?WavesPerEu = null, + unsafeFpAtomics: ?bool = null, + + pub const WavesPerEu = struct { + min: u32, + max: ?u32 = null, + }; +}; + +pub fn annotateFunction(ctx: *mlir.Context, op: *mlir.Operation, opts: FunctionOpts) void { + if (opts.kernel) op.setAttributeByName("rocdl.kernel", .unit(ctx)); + if (opts.reqdWorkGroupSize) |size| op.setAttributeByName("rocdl.reqd_work_group_size", .denseArray(ctx, .i32, &size)); + if (opts.flatWorkGroupSize) |size| op.setAttributeByName("rocdl.flat_work_group_size", .string(ctx, size)); + if (opts.maxFlatWorkGroupSize) |size| op.setAttributeByName("rocdl.max_flat_work_group_size", .int(ctx, .i32, size)); + if (opts.wavesPerEu) |waves| { + var buffer: [24]u8 = undefined; + const value = if (waves.max) |max| std.fmt.bufPrint(&buffer, "{d},{d}", .{ waves.min, max }) else std.fmt.bufPrint(&buffer, "{d}", .{waves.min}); + op.setAttributeByName("rocdl.waves_per_eu", .string(ctx, value catch unreachable)); + } + if (opts.unsafeFpAtomics) |enabled| op.setAttributeByName("rocdl.unsafe_fp_atomics", .boolean(ctx, enabled)); +} + +/// A missing flag preserves the target's default; false explicitly disables it. +pub const ModuleOpts = struct { + xnack: ?bool = null, + sramecc: ?bool = null, +}; + +pub fn annotateModule(ctx: *mlir.Context, op: *mlir.Operation, opts: ModuleOpts) void { + if (opts.xnack) |enabled| op.setAttributeByName("rocdl.xnack", .boolean(ctx, enabled)); + if (opts.sramecc) |enabled| op.setAttributeByName("rocdl.sramecc", .boolean(ctx, enabled)); +} + +pub const MemoryOpts = struct { + lastUse: bool = false, + noRemoteMemory: bool = false, + noFineGrainedMemory: bool = false, + ignoreDenormalMode: bool = false, +}; + +pub fn annotateMemory(ctx: *mlir.Context, op: *mlir.Operation, opts: MemoryOpts) void { + if (opts.lastUse) op.setAttributeByName("rocdl.last_use", .unit(ctx)); + if (opts.noRemoteMemory) op.setAttributeByName("rocdl.no_remote_memory", .unit(ctx)); + if (opts.noFineGrainedMemory) op.setAttributeByName("rocdl.no_fine_grained_memory", .unit(ctx)); + if (opts.ignoreDenormalMode) op.setAttributeByName("rocdl.ignore_denormal_mode", .unit(ctx)); +} diff --git a/mlir/dialects/rocdl/attributes.zig b/mlir/dialects/rocdl/attributes.zig new file mode 100644 index 00000000..9e1b9863 --- /dev/null +++ b/mlir/dialects/rocdl/attributes.zig @@ -0,0 +1,417 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const c = @import("c"); +const mlir = @import("mlir"); +const enums = @import("enums.zig"); + +pub const TargetAttr = @import("target.zig").TargetAttr; + +/// `#rocdl.buffer_oob_mode`. +pub const BufferOOBModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.buffer_oob_mode_flag`. +pub const BufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_atomic_cache_policy`. +pub const Gfx12AtomicCachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12AtomicCachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12AtomicCachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_cache_policy`. +pub const Gfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx942_cache_policy`. +pub const Gfx942CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx942CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx942CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx942CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx942CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_neg_modifier`. +pub const MFMANegModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMANegModifier; + pub const isAFn = c.mlirAttributeIsAROCDLMFMANegModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMANegModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMANegModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_perm_b`. +pub const MFMAPermBAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMAPermB; + pub const isAFn = c.mlirAttributeIsAROCDLMFMAPermB; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMAPermBAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMAPermBAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.matrix_format`. +pub const MatrixFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MatrixFormat; + pub const isAFn = c.mlirAttributeIsAROCDLMatrixFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMatrixFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.pre_gfx12_cache_policy`. +pub const PreGfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.PreGfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLPreGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLPreGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLPreGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.sched_group_mask`. +pub const SchedGroupMaskAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.SchedGroupMask; + pub const isAFn = c.mlirAttributeIsAROCDLSchedGroupMask; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLSchedGroupMaskAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLSchedGroupMaskAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.tbuffer_oob_mode_flag`. +pub const TBufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLTBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_c_modifier`. +pub const WMMACModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMACModifier; + pub const isAFn = c.mlirAttributeIsAROCDLWMMACModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMACModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale`. +pub const WMMAMatrixScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScale; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale_format`. +pub const WMMAMatrixScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScaleFormat; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const enum_attributes = .{ + BufferOOBModeAttr, + BufferOOBModeModuleFlagAttr, + Gfx12AtomicCachePolicyAttr, + Gfx12CachePolicyAttr, + Gfx942CachePolicyAttr, + MFMANegModifierAttr, + MFMAPermBAttr, + MatrixFormatAttr, + PreGfx12CachePolicyAttr, + SchedGroupMaskAttr, + TBufferOOBModeModuleFlagAttr, + WMMACModifierAttr, + WMMAMatrixScaleAttr, + WMMAMatrixScaleFormatAttr, +}; diff --git a/mlir/dialects/rocdl/capi.cc b/mlir/dialects/rocdl/capi.cc new file mode 100644 index 00000000..7a01bd56 --- /dev/null +++ b/mlir/dialects/rocdl/capi.cc @@ -0,0 +1,440 @@ +//===- capi.cc - C API for the rocdl dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" +#include "mlir/Dialect/LLVMIR/ROCDLDialect.h" +#include "mlir/IR/Diagnostics.h" + +// Introspection + +intptr_t mlirROCDLGetOperationCount(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return 0; + ctx->getOrLoadDialect(); + intptr_t count = 0; + for (auto op : ctx->getRegisteredOperations()) + if (op.getDialectNamespace() == "rocdl") + ++count; + return count; +} + +// Attributes + +bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeGfx12AtomicCachePolicy( + static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12AtomicCachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx942CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx942CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMANegModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMFMANegModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMANegModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeMFMAPermB(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMAPermBAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMatrixFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMatrixFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MatrixFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizePreGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::PreGfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLSchedGroupMaskAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeSchedGroupMask(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::SchedGroupMaskAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::TBufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMACModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + if (value > 65535) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMACModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMACModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScale(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScaleFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +// Target + +bool mlirAttributeIsAROCDLTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLTargetAttrGet(MlirContext context, int optLevel, + MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, + MlirAttribute flags, MlirAttribute link) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + for (auto s : {triple, chip, features, abi}) + if (!s.data && s.length) + return {nullptr}; + auto flagsValue = + llvm::dyn_cast_if_present(unwrap(flags)); + auto linkValue = llvm::dyn_cast_if_present(unwrap(link)); + if (unwrap(flags) && (!flagsValue || flagsValue.getContext() != ctx)) + return {nullptr}; + if (unwrap(link) && (!linkValue || linkValue.getContext() != ctx)) + return {nullptr}; + ctx->getOrLoadDialect(); + // The upstream custom getChecked builder delegates to unchecked Base::get. + // Verify first so invalid C API inputs return null instead of asserting. + if (mlir::failed(mlir::ROCDL::ROCDLTargetAttr::verifyInvariants( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, optLevel, + unwrap(triple), unwrap(chip), unwrap(features), unwrap(abi), + flagsValue, linkValue))) + return {nullptr}; + return wrap(mlir::ROCDL::ROCDLTargetAttr::get( + ctx, optLevel, unwrap(triple), unwrap(chip), unwrap(features), + unwrap(abi), flagsValue, linkValue)); +} + +int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} + +MlirStringRef mlirROCDLTargetAttrGetTriple(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getTriple()); +} + +MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} + +MlirStringRef mlirROCDLTargetAttrGetFeatures(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFeatures()); +} + +MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getAbi()); +} + +MlirAttribute mlirROCDLTargetAttrGetFlags(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFlags()); +} + +MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getLink()); +} + +bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, MlirStringRef flag) { + return llvm::cast(unwrap(attr)) + .hasFlag(unwrap(flag)); +} + +bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasWave64(); +} + +bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFastMath(); +} + +bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasDaz(); +} + +bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFiniteOnly(); +} + +bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasUnsafeMath(); +} + +bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .hasCorrectSqrt(); +} diff --git a/mlir/dialects/rocdl/capi.h b/mlir/dialects/rocdl/capi.h new file mode 100644 index 00000000..b73a948d --- /dev/null +++ b/mlir/dialects/rocdl/capi.h @@ -0,0 +1,140 @@ +//===- capi.h - C API for the rocdl dialect -----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_ROCDL_CAPI_H +#define MLIR_DIALECTS_ROCDL_CAPI_H + +#include "mlir-c/Dialect/ROCDL.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Introspection + +MLIR_CAPI_EXPORTED intptr_t mlirROCDLGetOperationCount(MlirContext context); + +// Attributes + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMFMANegModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMatrixFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLSchedGroupMaskAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMACModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr); + +// Target + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGet( + MlirContext context, int optLevel, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, MlirAttribute flags, + MlirAttribute link); +MLIR_CAPI_EXPORTED int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, + MlirStringRef flag); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_ROCDL_CAPI_H diff --git a/mlir/dialects/rocdl/enums.zig b/mlir/dialects/rocdl/enums.zig new file mode 100644 index 00000000..fde24ff7 --- /dev/null +++ b/mlir/dialects/rocdl/enums.zig @@ -0,0 +1,208 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/// ROCDL buffer out-of-bounds mode +pub const BufferOOBMode = enum(u32) { + any = 0, + relaxed = 1, + strict = 2, +}; + +/// gfx12 atomic cache policy bits +pub const Gfx12AtomicCachePolicy = enum(u32) { + none = 0, + keep_return = 1, + nt = 2, + cascade = 4, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12AtomicCachePolicy) Gfx12AtomicCachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12AtomicCachePolicy, value: Gfx12AtomicCachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx12 cache policy bits +pub const Gfx12CachePolicy = enum(u32) { + none = 0, + nt = 1, + ht = 2, + lu = 3, + nt_rt = 4, + rt_nt = 5, + nt_ht = 6, + nt_wb = 7, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + swz = 64, + scal = 2048, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12CachePolicy) Gfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12CachePolicy, value: Gfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx942 buffer cache policy bits +pub const Gfx942CachePolicy = enum(u32) { + none = 0, + sc0 = 1, + nt = 2, + swz = 8, + sc1 = 16, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx942CachePolicy) Gfx942CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx942CachePolicy, value: Gfx942CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// negation modifier bitfield for gfx94x double-precision MFMA +pub const MFMANegModifier = enum(u32) { + none = 0, + neg_a = 1, + neg_b = 2, + neg_c = 4, + _, + + pub fn combine(values: []const MFMANegModifier) MFMANegModifier { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: MFMANegModifier, value: MFMANegModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// permutations of the lanes storing B in an MFMA +pub const MFMAPermB = enum(u32) { + none = 0, + bcast_first_32 = 1, + bcast_second_32 = 2, + rotate_16_right = 3, + bcast_first_16 = 4, + bcast_second_16 = 5, + bcast_third_16 = 6, + bcast_fourth_16 = 7, +}; + +/// matrix operand formats selected by scaled MFMA/WMMA format fields +pub const MatrixFormat = enum(u32) { + fp8_e4m3 = 0, + fp8_e5m2 = 1, + fp6_e2m3 = 2, + fp6_e3m2 = 3, + fp4_e2m1 = 4, +}; + +/// pre-gfx12 buffer cache policy bits +pub const PreGfx12CachePolicy = enum(u32) { + none = 0, + glc = 1, + slc = 2, + dlc = 4, + swz = 8, + scc = 16, + all = 23, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const PreGfx12CachePolicy) PreGfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: PreGfx12CachePolicy, value: PreGfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// instruction type mask for scheduling barriers +pub const SchedGroupMask = enum(u32) { + none = 0, + non_mem_non_sideeffect = 1, + valu = 2, + salu = 4, + mfma_wmma = 8, + all_vmem = 16, + vmem_read = 32, + vmem_write = 64, + all_ds = 128, + ds_read = 256, + ds_write = 512, + transcendental = 1024, + ldsdma = 2048, + all = 4095, + _, + + pub fn combine(values: []const SchedGroupMask) SchedGroupMask { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: SchedGroupMask, value: SchedGroupMask) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// WMMA C operand modifiers +pub const WMMACModifier = enum(u16) { + none = 0, + neg = 1, + abs = 2, + neg_abs = 3, + _, + + pub fn combine(values: []const WMMACModifier) WMMACModifier { + var bits: u16 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: WMMACModifier, value: WMMACModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// matrix scale row selector +pub const WMMAMatrixScale = enum(u32) { + row0 = 0, + row1 = 1, +}; + +/// matrix scale exponent formats +pub const WMMAMatrixScaleFormat = enum(u32) { + e8 = 0, + e5m3 = 1, + e4m3 = 2, +}; diff --git a/mlir/dialects/rocdl/ops.zig b/mlir/dialects/rocdl/ops.zig new file mode 100644 index 00000000..b18a33cf --- /dev/null +++ b/mlir/dialects/rocdl/ops.zig @@ -0,0 +1,8600 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// `rocdl.asyncmark`. Mark the end of a group of asynchronous operations +pub fn asyncmark( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.asyncmark", .{ + .location = location, + }); +} + +/// `rocdl.ballot`. Vote across thread group +pub fn ballot( + ctx: *mlir.Context, + pred: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ballot", .{ + .operands = .{ .flat = &.{pred} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.barrier`. +pub fn barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.barrier", .{ + .location = location, + }); +} + +/// `rocdl.cluster.id.x`. +pub fn cluster_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.y`. +pub fn cluster_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.z`. +pub fn cluster_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b128`. +pub fn cluster_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b32`. +pub fn cluster_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b64`. +pub fn cluster_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b8`. +pub fn cluster_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.x`. +pub fn cluster_workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.y`. +pub fn cluster_workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.z`. +pub fn cluster_workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cos`. +pub fn cos( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cos", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.f32.bf8`. Convert bf8 to f32 +pub fn cvt_f32_bf8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.bf8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.f32.fp8`. Convert fp8 to f32 +pub fn cvt_f32_fp8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.fp8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.bf8.f32`. Convert two f32's to bf8 +pub fn cvt_pk_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.bf8`. Convert packed bf8 to packed f32 +pub fn cvt_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.bf8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.fp8`. Convert packed fp8 to packed f32 +pub fn cvt_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.fp8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.fp8.f32`. Convert two f32's to fp8 +pub fn cvt_pk_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pkrtz`. Convert two f32 input into a vector<2xf16> +pub fn cvt_pkrtz( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.pkrtz", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.bf6`. Scales 16 bf6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.fp6`. Scales 16 fp6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.bf6`. Scales 16 bf6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.fp6`. Scales 16 fp6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.bf6`. Scales 16 bf6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.fp6`. Scales 16 fp6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.bf8`. Scales 8 bf8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp4`. Scales 8 fp4 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp8`. Scales 8 fp8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.bf8`. Scales 8 bf8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp4`. Scales 8 fp4 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp8`. Scales 8 fp8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.bf8`. Scales 8 bf8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp4`. Scales 8 fp4 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp8`. Scales 8 fp8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.bf6.f32`. Scale and convert two vector<16xf32> to 32 packed bf6 +pub fn cvt_scalef32_2xpk16_bf6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.fp6.f32`. Scale and convert two vector<16xf32> to 32 packed fp6 +pub fn cvt_scalef32_2xpk16_fp6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.bf8`. Scaled convert bf8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_bf8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.bf8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.fp8`. Scaled convert fp8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_fp8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.fp8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.bf8`. Scaled convert bf8 from packed vector to f32 +pub fn cvt_scalef32_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.fp8`. Scaled convert fp8 from packed vector to f32 +pub fn cvt_scalef32_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.bf8`. Scaled convert two bf8to two bf16 +pub fn cvt_scalef32_pk_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp4`. Scale and convert two packed fp4 to packed bf16 +pub fn cvt_scalef32_pk_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp8`. Scaled convert two fp8to two bf16 +pub fn cvt_scalef32_pk_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.bf16`. Scaled convert two bf16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f16`. Scaled convert two f16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f32`. Scaled convert two f32 to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.bf8`. Scaled convert two bf8to two f16 +pub fn cvt_scalef32_pk_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp4`. Scale and convert two packed fp4 to packed f16 +pub fn cvt_scalef32_pk_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp8`. Scaled convert two fp8to two f16 +pub fn cvt_scalef32_pk_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.bf8`. Scaled convert two bf8to two f32 +pub fn cvt_scalef32_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp4`. Scale and convert two packed fp4 to packed f32 +pub fn cvt_scalef32_pk_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp8`. Scaled convert two fp8to two f32 +pub fn cvt_scalef32_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f16`. Scale and convert two f16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f32`. Scale and convert two f32 values to two packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.bf16`. Scaled convert two bf16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f16`. Scaled convert two f16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f32`. Scaled convert two f32 to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.bf6`. Scale and convert packed bf6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.fp6`. Scale and convert packed fp6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.bf6`. Scale and convert packed bf6 to packed f16 +pub fn cvt_scalef32_pk32_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.fp6`. Scale and convert packed fp6 to packed f16 +pub fn cvt_scalef32_pk32_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.bf6`. Scale and convert packed bf6 to packed f32 +pub fn cvt_scalef32_pk32_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.fp6`. Scale and convert packed fp6 to packed f32 +pub fn cvt_scalef32_pk32_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.bf16`. Scaled convert bf16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f16`. Scaled convert f16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f32`. Scaled convert f32to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.bf16`. Scaled convert bf16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f16`. Scaled convert f16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f32`. Scaled convert f32to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f16`. Scale and convert two f16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f32`. Scale and convert two f32 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.sr.bf8.f32`. Convert f32 to bf8, stochiastic rounding +pub fn cvt_sr_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.sr.fp8.f32`. Convert f32 to fp8, stochiastic rounding +pub fn cvt_sr_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.bf8`. +pub fn dot4_f32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.fp8`. +pub fn dot4_f32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.bf8`. +pub fn dot4_f32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.fp8`. +pub fn dot4_f32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds.atomic.async.barrier.arrive.b64`. +pub fn ds_atomic_async_barrier_arrive_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.async.barrier.arrive.b64", .{ + .operands = .{ .flat = &.{barrierPtr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.atomic.barrier.arrive.rtn.b64`. +pub fn ds_atomic_barrier_arrive_rtn_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + val: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.barrier.arrive.rtn.b64", .{ + .operands = .{ .flat = &.{ barrierPtr, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr16.b128`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr16_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr16.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr4.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr6.b96`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr8.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr16.b64`. +pub fn ds_read_tr16_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr16.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr4.b64`. +pub fn ds_read_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr6.b96`. +pub fn ds_read_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr8.b64`. +pub fn ds_read_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds_bpermute`. +pub fn ds_bpermute( + ctx: *mlir.Context, + index: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_bpermute", .{ + .operands = .{ .flat = &.{ index, src } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds_swizzle`. +pub fn ds_swizzle( + ctx: *mlir.Context, + src: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_swizzle", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp`. +pub fn exp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp2`. +pub fn exp2( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp2", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2`. +pub fn fdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fdot2.bf16.bf16`. +pub fn fdot2_bf16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.bf16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f16.f16`. +pub fn fdot2_f16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.f16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f32.bf16`. +pub fn fdot2_f32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2.f32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.flat.prefetch`. +pub fn flat_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.flat.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fmed3`. Median of three float/half values +pub fn fmed3( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fmed3", .{ + .operands = .{ .flat = &.{ src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.global.load.async.lds`. Version of rocdl.load.async.to.lds specialized to global pointers +pub fn global_load_async_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b128`. +pub fn global_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b32`. +pub fn global_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b64`. +pub fn global_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b8`. +pub fn global_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.lds`. +pub fn global_load_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b128`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr4.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr6.b96`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.prefetch`. +pub fn global_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b128`. +pub fn global_store_async_from_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b32`. +pub fn global_store_async_from_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b64`. +pub fn global_store_async_from_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b8`. +pub fn global_store_async_from_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.iglp.opt`. +pub fn iglp_opt( + ctx: *mlir.Context, + variant: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "variant", variant)); + return mlir.Operation.make(ctx, "rocdl.iglp.opt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.async.to.lds`. Gathering load to LDS that requires explicit async memory tracking +pub fn load_async_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.async.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.to.lds`. +pub fn load_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.log`. +pub fn log( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.log", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.make.buffer.rsrc`. +pub fn make_buffer_rsrc( + ctx: *mlir.Context, + base: *const mlir.Value, + stride: *const mlir.Value, + numRecords: *const mlir.Value, + flags: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.make.buffer.rsrc", .{ + .operands = .{ .flat = &.{ base, stride, numRecords, flags } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.mbcnt.hi`. +pub fn mbcnt_hi( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.hi", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mbcnt.lo`. +pub fn mbcnt_lo( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.lo", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16bf16.1k`. +pub fn mfma_f32_16x16x16bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16f16`. +pub fn mfma_f32_16x16x16f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x1f32`. +pub fn mfma_f32_16x16x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x2bf16`. +pub fn mfma_f32_16x16x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf16`. +pub fn mfma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.bf8`. +pub fn mfma_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.fp8`. +pub fn mfma_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.f16`. +pub fn mfma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.bf8`. +pub fn mfma_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.fp8`. +pub fn mfma_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4bf16.1k`. +pub fn mfma_f32_16x16x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f16`. +pub fn mfma_f32_16x16x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f32`. +pub fn mfma_f32_16x16x4f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8.xf32`. +pub fn mfma_f32_16x16x8_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8bf16`. +pub fn mfma_f32_16x16x8bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf16`. +pub fn mfma_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.bf8`. +pub fn mfma_f32_32x32x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.fp8`. +pub fn mfma_f32_32x32x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.f16`. +pub fn mfma_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.bf8`. +pub fn mfma_f32_32x32x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.fp8`. +pub fn mfma_f32_32x32x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x1f32`. +pub fn mfma_f32_32x32x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2bf16`. +pub fn mfma_f32_32x32x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2f32`. +pub fn mfma_f32_32x32x2f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4.xf32`. +pub fn mfma_f32_32x32x4_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16`. +pub fn mfma_f32_32x32x4bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16.1k`. +pub fn mfma_f32_32x32x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4f16`. +pub fn mfma_f32_32x32x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8bf16.1k`. +pub fn mfma_f32_32x32x8bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8f16`. +pub fn mfma_f32_32x32x8f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x1f32`. +pub fn mfma_f32_4x4x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x2bf16`. +pub fn mfma_f32_4x4x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4bf16.1k`. +pub fn mfma_f32_4x4x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4f16`. +pub fn mfma_f32_4x4x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.16x16x4f64`. +pub fn mfma_f64_16x16x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.16x16x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.4x4x4f64`. +pub fn mfma_f64_4x4x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.4x4x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x16i8`. +pub fn mfma_i32_16x16x16i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x16i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x32.i8`. +pub fn mfma_i32_16x16x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x4i8`. +pub fn mfma_i32_16x16x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x64.i8`. +pub fn mfma_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x16.i8`. +pub fn mfma_i32_32x32x16_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x16.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x32.i8`. +pub fn mfma_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x4i8`. +pub fn mfma_i32_32x32x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x8i8`. +pub fn mfma_i32_32x32x8i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x8i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.4x4x4i8`. +pub fn mfma_i32_4x4x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.4x4x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.16x16x128.f8f6f4`. +pub fn mfma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.32x32x64.f8f6f4`. +pub fn mfma_scale_f32_32x32x64_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.32x32x64.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.swap`. +pub fn permlane16_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.var`. +pub fn permlane16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane32.swap`. +pub fn permlane32_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane32.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16`. +pub fn permlanex16( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16", .{ + .operands = .{ .flat = &.{ old, src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16.var`. +pub fn permlanex16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ptr.s.buffer.load`. +pub fn ptr_s_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ptr.s.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.cmpswap`. +pub fn raw_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fadd`. +pub fn raw_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fmax`. +pub fn raw_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.smax`. +pub fn raw_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.umin`. +pub fn raw_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.load`. +pub fn raw_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.store`. +pub fn raw_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.cmpswap`. +pub fn raw_ptr_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fadd`. +pub fn raw_ptr_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fmax`. +pub fn raw_ptr_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.smax`. +pub fn raw_ptr_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.umin`. +pub fn raw_ptr_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load`. +pub fn raw_ptr_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.async.lds`. Async variant of raw.ptr.buffer.load.lds +pub fn raw_ptr_buffer_load_async_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.async.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.lds`. +pub fn raw_ptr_buffer_load_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.store`. +pub fn raw_ptr_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.rcp`. +pub fn rcp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rcp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readfirstlane`. Get the value in first active lane. +pub fn readfirstlane( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readfirstlane", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readlane`. Get the value in the specific lane. +pub fn readlane( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readlane", .{ + .operands = .{ .flat = &.{ src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.rsq`. +pub fn rsq( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rsq", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.barrier`. +pub fn s_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier", .{ + .location = location, + }); +} + +/// `rocdl.s.barrier.init`. +pub fn s_barrier_init( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.init", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.join`. +pub fn s_barrier_join( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier.join", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.s.barrier.leave`. +pub fn s_barrier_leave( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.leave", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal`. +pub fn s_barrier_signal( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.isfirst`. +pub fn s_barrier_signal_isfirst( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.isfirst", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.var`. +pub fn s_barrier_signal_var( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.var", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.wait`. +pub fn s_barrier_wait( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.barrier.state`. +pub fn s_get_barrier_state( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.get.barrier.state", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.named.barrier.state`. +pub fn s_get_named_barrier_state( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.get.named.barrier.state", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.nop`. +pub fn s_nop( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.nop", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.setprio`. +pub fn s_setprio( + ctx: *mlir.Context, + priority: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "priority", priority)); + return mlir.Operation.make(ctx, "rocdl.s.setprio", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.sleep`. +pub fn s_sleep( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.sleep", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.asynccnt`. Wait until ASYNCCNT is less than or equal to `count` +pub fn s_wait_asynccnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.asynccnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.dscnt`. Wait until DSCNT is less than or equal to `count` +pub fn s_wait_dscnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.dscnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.expcnt`. Wait until EXPCNT is less than or equal to `count` +pub fn s_wait_expcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.expcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.loadcnt`. Wait until LOADCNT is less than or equal to `count` +pub fn s_wait_loadcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.loadcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.storecnt`. Wait until STORECNT is less than or equal to `count` +pub fn s_wait_storecnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.storecnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.tensorcnt`. Wait until TENSORCNT is less than or equal to `count` +pub fn s_wait_tensorcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.tensorcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.waitcnt`. +pub fn s_waitcnt( + ctx: *mlir.Context, + bitfield: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "bitfield", bitfield)); + return mlir.Operation.make(ctx, "rocdl.s.waitcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wakeup.barrier`. +pub fn s_wakeup_barrier( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.wakeup.barrier", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.sched.barrier`. +pub fn sched_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + return mlir.Operation.make(ctx, "rocdl.sched.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sched.group.barrier`. +pub fn sched_group_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + size: *const mlir.Attribute, + groupId: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "groupId", groupId)); + return mlir.Operation.make(ctx, "rocdl.sched.group.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot2`. +pub fn sdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot4`. +pub fn sdot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot8`. +pub fn sdot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sin`. +pub fn sin( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sin", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.bf8`. +pub fn smfmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.fp8`. +pub fn smfmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.bf8`. +pub fn smfmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.fp8`. +pub fn smfmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.bf16`. +pub fn smfmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.f16`. +pub fn smfmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf16`. +pub fn smfmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.bf8`. +pub fn smfmac_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.fp8`. +pub fn smfmac_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.f16`. +pub fn smfmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.bf8`. +pub fn smfmac_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.fp8`. +pub fn smfmac_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.bf16`. +pub fn smfmac_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.f16`. +pub fn smfmac_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf16`. +pub fn smfmac_f32_32x32x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.bf8`. +pub fn smfmac_f32_32x32x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.fp8`. +pub fn smfmac_f32_32x32x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.f16`. +pub fn smfmac_f32_32x32x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.bf8`. +pub fn smfmac_f32_32x32x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.fp8`. +pub fn smfmac_f32_32x32x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.bf8`. +pub fn smfmac_f32_32x32x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.fp8`. +pub fn smfmac_f32_32x32x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.bf8`. +pub fn smfmac_f32_32x32x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.fp8`. +pub fn smfmac_f32_32x32x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x128.i8`. +pub fn smfmac_i32_16x16x128_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x128.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x64.i8`. +pub fn smfmac_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x32.i8`. +pub fn smfmac_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x64.i8`. +pub fn smfmac_i32_32x32x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sqrt`. +pub fn sqrt( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sqrt", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.sudot4`. +pub fn sudot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sudot8`. +pub fn sudot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x32.bf16`. +pub fn swmmac_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x64.bf16`. +pub fn swmmac_bf16_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16f32.16x16x64.bf16`. +pub fn swmmac_bf16f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.bf8`. +pub fn swmmac_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.fp8`. +pub fn swmmac_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.bf8`. +pub fn swmmac_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.fp8`. +pub fn swmmac_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x32.f16`. +pub fn swmmac_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x64.f16`. +pub fn swmmac_f16_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.bf8`. +pub fn swmmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.fp8`. +pub fn swmmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.bf8`. +pub fn swmmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.fp8`. +pub fn swmmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf16`. +pub fn swmmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.bf8`. +pub fn swmmac_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.fp8`. +pub fn swmmac_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.f16`. +pub fn swmmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.bf8`. +pub fn swmmac_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.fp8`. +pub fn swmmac_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.bf16`. +pub fn swmmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.f16`. +pub fn swmmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x128.iu8`. +pub fn swmmac_i32_16x16x128_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x128.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu4`. +pub fn swmmac_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu8`. +pub fn swmmac_i32_16x16x32_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x64.iu4`. +pub fn swmmac_i32_16x16x64_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x64.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tanh`. +pub fn tanh( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.tanh", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.tensor.load.to.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_load_to_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.load.to.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tensor.store.from.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_store_from_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.store.from.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot2`. +pub fn udot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot4`. +pub fn udot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot8`. +pub fn udot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.update.dpp`. +pub fn update_dpp( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + dppCtrl: *const mlir.Attribute, + rowMask: *const mlir.Attribute, + bankMask: *const mlir.Attribute, + boundCtrl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dppCtrl", dppCtrl)); + attributes.appendAssumeCapacity(.named(ctx, "rowMask", rowMask)); + attributes.appendAssumeCapacity(.named(ctx, "bankMask", bankMask)); + attributes.appendAssumeCapacity(.named(ctx, "boundCtrl", boundCtrl)); + return mlir.Operation.make(ctx, "rocdl.update.dpp", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wait.asyncmark`. Wait until N or fewer async operation groups are unexecuted +pub fn wait_asyncmark( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.wait.asyncmark", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wave.barrier`. +pub fn wave_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wave.barrier", .{ + .location = location, + }); +} + +/// `rocdl.wave.id`. +pub fn wave_id( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wave.id", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wavefrontsize`. +pub fn wavefrontsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wavefrontsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x16.bf16`. +pub fn wmma_bf16_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x32.bf16`. +pub fn wmma_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16f32.16x16x32.bf16`. +pub fn wmma_bf16f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_bf8`. +pub fn wmma_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_fp8`. +pub fn wmma_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_bf8`. +pub fn wmma_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_fp8`. +pub fn wmma_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x16.f16`. +pub fn wmma_f16_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x32.f16`. +pub fn wmma_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_bf8`. +pub fn wmma_f16_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_fp8`. +pub fn wmma_f16_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_bf8`. +pub fn wmma_f16_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_fp8`. +pub fn wmma_f16_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_bf8`. +pub fn wmma_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_fp8`. +pub fn wmma_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_bf8`. +pub fn wmma_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_fp8`. +pub fn wmma_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf16`. +pub fn wmma_f32_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_bf8`. +pub fn wmma_f32_16x16x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_fp8`. +pub fn wmma_f32_16x16x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.f16`. +pub fn wmma_f32_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_bf8`. +pub fn wmma_f32_16x16x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_fp8`. +pub fn wmma_f32_16x16x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.bf16`. +pub fn wmma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.f16`. +pub fn wmma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x4.f32`. +pub fn wmma_f32_16x16x4_f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x4.f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_bf8`. +pub fn wmma_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_fp8`. +pub fn wmma_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_bf8`. +pub fn wmma_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_fp8`. +pub fn wmma_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu4`. +pub fn wmma_i32_16x16x16_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu8`. +pub fn wmma_i32_16x16x16_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x32.iu4`. +pub fn wmma_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x64.iu8`. +pub fn wmma_i32_16x16x64_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x64.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.16x16x128.f8f6f4`. +pub fn wmma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.32x16x128.f4`. +pub fn wmma_scale_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.16x16x128.f8f6f4`. +pub fn wmma_scale16_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.32x16x128.f4`. +pub fn wmma_scale16_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.x`. +pub fn workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.y`. +pub fn workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.z`. +pub fn workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.x`. +pub fn workitem_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.y`. +pub fn workitem_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.z`. +pub fn workitem_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/rocdl/rocdl.zig b/mlir/dialects/rocdl/rocdl.zig new file mode 100644 index 00000000..456ad494 --- /dev/null +++ b/mlir/dialects/rocdl/rocdl.zig @@ -0,0 +1,56 @@ +//! LLVM ROCDL operations, AMDGPU attributes, and GPU target configuration. +//! All operation builders from the pinned LLVM revision are available in `ops`. + +const mlir = @import("mlir"); + +pub const types = @import("types.zig"); +pub const annotations = @import("annotations.zig"); +pub const AddressSpace = types.AddressSpace; +pub const pointerType = types.pointerType; + +pub const ops = @import("ops.zig"); +pub const enums = @import("enums.zig"); +pub const attributes = @import("attributes.zig"); + +pub const BufferOOBModeAttr = attributes.BufferOOBModeAttr; +pub const BufferOOBModeModuleFlagAttr = attributes.BufferOOBModeModuleFlagAttr; +pub const Gfx12AtomicCachePolicyAttr = attributes.Gfx12AtomicCachePolicyAttr; +pub const Gfx12CachePolicyAttr = attributes.Gfx12CachePolicyAttr; +pub const Gfx942CachePolicyAttr = attributes.Gfx942CachePolicyAttr; +pub const MFMANegModifierAttr = attributes.MFMANegModifierAttr; +pub const MFMAPermBAttr = attributes.MFMAPermBAttr; +pub const MatrixFormatAttr = attributes.MatrixFormatAttr; +pub const PreGfx12CachePolicyAttr = attributes.PreGfx12CachePolicyAttr; +pub const SchedGroupMaskAttr = attributes.SchedGroupMaskAttr; +pub const TBufferOOBModeModuleFlagAttr = attributes.TBufferOOBModeModuleFlagAttr; +pub const TargetAttr = attributes.TargetAttr; +pub const WMMACModifierAttr = attributes.WMMACModifierAttr; +pub const WMMAMatrixScaleAttr = attributes.WMMAMatrixScaleAttr; +pub const WMMAMatrixScaleFormatAttr = attributes.WMMAMatrixScaleFormatAttr; + +pub const BufferOOBMode = enums.BufferOOBMode; +pub const Gfx12AtomicCachePolicy = enums.Gfx12AtomicCachePolicy; +pub const Gfx12CachePolicy = enums.Gfx12CachePolicy; +pub const Gfx942CachePolicy = enums.Gfx942CachePolicy; +pub const MFMANegModifier = enums.MFMANegModifier; +pub const MFMAPermB = enums.MFMAPermB; +pub const MatrixFormat = enums.MatrixFormat; +pub const PreGfx12CachePolicy = enums.PreGfx12CachePolicy; +pub const SchedGroupMask = enums.SchedGroupMask; +pub const WMMACModifier = enums.WMMACModifier; +pub const WMMAMatrixScale = enums.WMMAMatrixScale; +pub const WMMAMatrixScaleFormat = enums.WMMAMatrixScaleFormat; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("rocdl"); +} + +/// Converts two packed FP8 values from the selected word to vector<2xf32>. +pub fn cvtPkF32Fp8( + ctx: *mlir.Context, + bits: *const mlir.Value, + high: bool, + location: *const mlir.Location, +) *mlir.Operation { + return ops.cvt_pk_f32_fp8(ctx, bits, .vector(&.{2}, .float(ctx, .f32)), .boolean(ctx, high), location); +} diff --git a/mlir/dialects/rocdl/target.zig b/mlir/dialects/rocdl/target.zig new file mode 100644 index 00000000..30311dc5 --- /dev/null +++ b/mlir/dialects/rocdl/target.zig @@ -0,0 +1,100 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +/// GPU compilation target, `#rocdl.target`. +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAROCDLTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + optLevel: i32 = 2, + triple: []const u8 = "amdgcn-amd-amdhsa", + chip: []const u8 = "gfx900", + features: []const u8 = "", + abi: []const u8 = "600", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTargetAttrGet( + ctx.ptr(), + args.optLevel, + mlir.stringRef(args.triple), + mlir.stringRef(args.chip), + mlir.stringRef(args.features), + mlir.stringRef(args.abi), + .{ .ptr = if (args.flags) |a| @constCast(a.ptr().ptr) else null }, + .{ .ptr = if (args.link) |a| @constCast(a.ptr().ptr) else null }, + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn optLevel(self: *const Self) i32 { + return c.mlirROCDLTargetAttrGetOptLevel(self.ptr()); + } + + pub fn triple(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetTriple(self.ptr())); + } + + pub fn chip(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetChip(self.ptr())); + } + + pub fn features(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetFeatures(self.ptr())); + } + + pub fn abi(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetAbi(self.ptr())); + } + + pub fn flags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetFlags(self.ptr()).ptr); + } + + pub fn link(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetLink(self.ptr()).ptr); + } + + pub fn hasFlag(self: *const Self, flag: []const u8) bool { + return c.mlirROCDLTargetAttrHasFlag(self.ptr(), mlir.stringRef(flag)); + } + + pub fn hasWave64(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasWave64(self.ptr()); + } + + pub fn hasFastMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFastMath(self.ptr()); + } + + pub fn hasDaz(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasDaz(self.ptr()); + } + + pub fn hasFiniteOnly(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFiniteOnly(self.ptr()); + } + + pub fn hasUnsafeMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasUnsafeMath(self.ptr()); + } + + pub fn hasCorrectSqrt(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasCorrectSqrt(self.ptr()); + } +}; + +fn string(ref: c.MlirStringRef) []const u8 { + return if (ref.length == 0) "" else mlir.string(ref); +} diff --git a/mlir/dialects/rocdl/test.zig b/mlir/dialects/rocdl/test.zig new file mode 100644 index 00000000..81f0b646 --- /dev/null +++ b/mlir/dialects/rocdl/test.zig @@ -0,0 +1,592 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); + +const operation_names: []const []const u8 = &.{ + "rocdl.asyncmark", + "rocdl.ballot", + "rocdl.barrier", + "rocdl.cluster.id.x", + "rocdl.cluster.id.y", + "rocdl.cluster.id.z", + "rocdl.cluster.load.async.to.lds.b128", + "rocdl.cluster.load.async.to.lds.b32", + "rocdl.cluster.load.async.to.lds.b64", + "rocdl.cluster.load.async.to.lds.b8", + "rocdl.cluster.workgroup.id.x", + "rocdl.cluster.workgroup.id.y", + "rocdl.cluster.workgroup.id.z", + "rocdl.cos", + "rocdl.cvt.f32.bf8", + "rocdl.cvt.f32.fp8", + "rocdl.cvt.pk.bf8.f32", + "rocdl.cvt.pk.f32.bf8", + "rocdl.cvt.pk.f32.fp8", + "rocdl.cvt.pk.fp8.f32", + "rocdl.cvt.pkrtz", + "rocdl.cvt.scale.pk16.bf16.bf6", + "rocdl.cvt.scale.pk16.bf16.fp6", + "rocdl.cvt.scale.pk16.f16.bf6", + "rocdl.cvt.scale.pk16.f16.fp6", + "rocdl.cvt.scale.pk16.f32.bf6", + "rocdl.cvt.scale.pk16.f32.fp6", + "rocdl.cvt.scale.pk8.bf16.bf8", + "rocdl.cvt.scale.pk8.bf16.fp4", + "rocdl.cvt.scale.pk8.bf16.fp8", + "rocdl.cvt.scale.pk8.f16.bf8", + "rocdl.cvt.scale.pk8.f16.fp4", + "rocdl.cvt.scale.pk8.f16.fp8", + "rocdl.cvt.scale.pk8.f32.bf8", + "rocdl.cvt.scale.pk8.f32.fp4", + "rocdl.cvt.scale.pk8.f32.fp8", + "rocdl.cvt.scalef32.2xpk16.bf6.f32", + "rocdl.cvt.scalef32.2xpk16.fp6.f32", + "rocdl.cvt.scalef32.f16.bf8", + "rocdl.cvt.scalef32.f16.fp8", + "rocdl.cvt.scalef32.f32.bf8", + "rocdl.cvt.scalef32.f32.fp8", + "rocdl.cvt.scalef32.pk.bf16.bf8", + "rocdl.cvt.scalef32.pk.bf16.fp4", + "rocdl.cvt.scalef32.pk.bf16.fp8", + "rocdl.cvt.scalef32.pk.bf8.bf16", + "rocdl.cvt.scalef32.pk.bf8.f16", + "rocdl.cvt.scalef32.pk.bf8.f32", + "rocdl.cvt.scalef32.pk.f16.bf8", + "rocdl.cvt.scalef32.pk.f16.fp4", + "rocdl.cvt.scalef32.pk.f16.fp8", + "rocdl.cvt.scalef32.pk.f32.bf8", + "rocdl.cvt.scalef32.pk.f32.fp4", + "rocdl.cvt.scalef32.pk.f32.fp8", + "rocdl.cvt.scalef32.pk.fp4.bf16", + "rocdl.cvt.scalef32.pk.fp4.f16", + "rocdl.cvt.scalef32.pk.fp4.f32", + "rocdl.cvt.scalef32.pk.fp8.bf16", + "rocdl.cvt.scalef32.pk.fp8.f16", + "rocdl.cvt.scalef32.pk.fp8.f32", + "rocdl.cvt.scalef32.pk16.bf6.bf16", + "rocdl.cvt.scalef32.pk16.bf6.f16", + "rocdl.cvt.scalef32.pk16.bf6.f32", + "rocdl.cvt.scalef32.pk16.fp6.bf16", + "rocdl.cvt.scalef32.pk16.fp6.f16", + "rocdl.cvt.scalef32.pk16.fp6.f32", + "rocdl.cvt.scalef32.pk32.bf16.bf6", + "rocdl.cvt.scalef32.pk32.bf16.fp6", + "rocdl.cvt.scalef32.pk32.bf6.bf16", + "rocdl.cvt.scalef32.pk32.bf6.f16", + "rocdl.cvt.scalef32.pk32.f16.bf6", + "rocdl.cvt.scalef32.pk32.f16.fp6", + "rocdl.cvt.scalef32.pk32.f32.bf6", + "rocdl.cvt.scalef32.pk32.f32.fp6", + "rocdl.cvt.scalef32.pk32.fp6.bf16", + "rocdl.cvt.scalef32.pk32.fp6.f16", + "rocdl.cvt.scalef32.pk8.bf8.bf16", + "rocdl.cvt.scalef32.pk8.bf8.f16", + "rocdl.cvt.scalef32.pk8.bf8.f32", + "rocdl.cvt.scalef32.pk8.fp4.bf16", + "rocdl.cvt.scalef32.pk8.fp4.f16", + "rocdl.cvt.scalef32.pk8.fp4.f32", + "rocdl.cvt.scalef32.pk8.fp8.bf16", + "rocdl.cvt.scalef32.pk8.fp8.f16", + "rocdl.cvt.scalef32.pk8.fp8.f32", + "rocdl.cvt.scalef32.sr.bf8.bf16", + "rocdl.cvt.scalef32.sr.bf8.f16", + "rocdl.cvt.scalef32.sr.bf8.f32", + "rocdl.cvt.scalef32.sr.fp8.bf16", + "rocdl.cvt.scalef32.sr.fp8.f16", + "rocdl.cvt.scalef32.sr.fp8.f32", + "rocdl.cvt.scalef32.sr.pk.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk.fp4.f16", + "rocdl.cvt.scalef32.sr.pk.fp4.f32", + "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f32", + "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f32", + "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f32", + "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f32", + "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f32", + "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f32", + "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f32", + "rocdl.cvt.sr.bf8.f32", + "rocdl.cvt.sr.fp8.f32", + "rocdl.dot4.f32.bf8.bf8", + "rocdl.dot4.f32.bf8.fp8", + "rocdl.dot4.f32.fp8.bf8", + "rocdl.dot4.f32.fp8.fp8", + "rocdl.ds.atomic.async.barrier.arrive.b64", + "rocdl.ds.atomic.barrier.arrive.rtn.b64", + "rocdl.ds.load.tr16.b128", + "rocdl.ds.load.tr4.b64", + "rocdl.ds.load.tr6.b96", + "rocdl.ds.load.tr8.b64", + "rocdl.ds.read.tr16.b64", + "rocdl.ds.read.tr4.b64", + "rocdl.ds.read.tr6.b96", + "rocdl.ds.read.tr8.b64", + "rocdl.ds_bpermute", + "rocdl.ds_swizzle", + "rocdl.exp", + "rocdl.exp2", + "rocdl.fdot2", + "rocdl.fdot2.bf16.bf16", + "rocdl.fdot2.f16.f16", + "rocdl.fdot2.f32.bf16", + "rocdl.flat.prefetch", + "rocdl.fmed3", + "rocdl.global.load.async.lds", + "rocdl.global.load.async.to.lds.b128", + "rocdl.global.load.async.to.lds.b32", + "rocdl.global.load.async.to.lds.b64", + "rocdl.global.load.async.to.lds.b8", + "rocdl.global.load.lds", + "rocdl.global.load.tr.b128", + "rocdl.global.load.tr.b64", + "rocdl.global.load.tr4.b64", + "rocdl.global.load.tr6.b96", + "rocdl.global.prefetch", + "rocdl.global.store.async.from.lds.b128", + "rocdl.global.store.async.from.lds.b32", + "rocdl.global.store.async.from.lds.b64", + "rocdl.global.store.async.from.lds.b8", + "rocdl.iglp.opt", + "rocdl.load.async.to.lds", + "rocdl.load.to.lds", + "rocdl.log", + "rocdl.make.buffer.rsrc", + "rocdl.mbcnt.hi", + "rocdl.mbcnt.lo", + "rocdl.mfma.f32.16x16x16bf16.1k", + "rocdl.mfma.f32.16x16x16f16", + "rocdl.mfma.f32.16x16x1f32", + "rocdl.mfma.f32.16x16x2bf16", + "rocdl.mfma.f32.16x16x32.bf16", + "rocdl.mfma.f32.16x16x32.bf8.bf8", + "rocdl.mfma.f32.16x16x32.bf8.fp8", + "rocdl.mfma.f32.16x16x32.f16", + "rocdl.mfma.f32.16x16x32.fp8.bf8", + "rocdl.mfma.f32.16x16x32.fp8.fp8", + "rocdl.mfma.f32.16x16x4bf16.1k", + "rocdl.mfma.f32.16x16x4f16", + "rocdl.mfma.f32.16x16x4f32", + "rocdl.mfma.f32.16x16x8.xf32", + "rocdl.mfma.f32.16x16x8bf16", + "rocdl.mfma.f32.32x32x16.bf16", + "rocdl.mfma.f32.32x32x16.bf8.bf8", + "rocdl.mfma.f32.32x32x16.bf8.fp8", + "rocdl.mfma.f32.32x32x16.f16", + "rocdl.mfma.f32.32x32x16.fp8.bf8", + "rocdl.mfma.f32.32x32x16.fp8.fp8", + "rocdl.mfma.f32.32x32x1f32", + "rocdl.mfma.f32.32x32x2bf16", + "rocdl.mfma.f32.32x32x2f32", + "rocdl.mfma.f32.32x32x4.xf32", + "rocdl.mfma.f32.32x32x4bf16", + "rocdl.mfma.f32.32x32x4bf16.1k", + "rocdl.mfma.f32.32x32x4f16", + "rocdl.mfma.f32.32x32x8bf16.1k", + "rocdl.mfma.f32.32x32x8f16", + "rocdl.mfma.f32.4x4x1f32", + "rocdl.mfma.f32.4x4x2bf16", + "rocdl.mfma.f32.4x4x4bf16.1k", + "rocdl.mfma.f32.4x4x4f16", + "rocdl.mfma.f64.16x16x4f64", + "rocdl.mfma.f64.4x4x4f64", + "rocdl.mfma.i32.16x16x16i8", + "rocdl.mfma.i32.16x16x32.i8", + "rocdl.mfma.i32.16x16x4i8", + "rocdl.mfma.i32.16x16x64.i8", + "rocdl.mfma.i32.32x32x16.i8", + "rocdl.mfma.i32.32x32x32.i8", + "rocdl.mfma.i32.32x32x4i8", + "rocdl.mfma.i32.32x32x8i8", + "rocdl.mfma.i32.4x4x4i8", + "rocdl.mfma.scale.f32.16x16x128.f8f6f4", + "rocdl.mfma.scale.f32.32x32x64.f8f6f4", + "rocdl.permlane16.swap", + "rocdl.permlane16.var", + "rocdl.permlane32.swap", + "rocdl.permlanex16", + "rocdl.permlanex16.var", + "rocdl.ptr.s.buffer.load", + "rocdl.raw.buffer.atomic.cmpswap", + "rocdl.raw.buffer.atomic.fadd", + "rocdl.raw.buffer.atomic.fmax", + "rocdl.raw.buffer.atomic.smax", + "rocdl.raw.buffer.atomic.umin", + "rocdl.raw.buffer.load", + "rocdl.raw.buffer.store", + "rocdl.raw.ptr.buffer.atomic.cmpswap", + "rocdl.raw.ptr.buffer.atomic.fadd", + "rocdl.raw.ptr.buffer.atomic.fmax", + "rocdl.raw.ptr.buffer.atomic.smax", + "rocdl.raw.ptr.buffer.atomic.umin", + "rocdl.raw.ptr.buffer.load", + "rocdl.raw.ptr.buffer.load.async.lds", + "rocdl.raw.ptr.buffer.load.lds", + "rocdl.raw.ptr.buffer.store", + "rocdl.rcp", + "rocdl.readfirstlane", + "rocdl.readlane", + "rocdl.rsq", + "rocdl.s.barrier", + "rocdl.s.barrier.init", + "rocdl.s.barrier.join", + "rocdl.s.barrier.leave", + "rocdl.s.barrier.signal", + "rocdl.s.barrier.signal.isfirst", + "rocdl.s.barrier.signal.var", + "rocdl.s.barrier.wait", + "rocdl.s.get.barrier.state", + "rocdl.s.get.named.barrier.state", + "rocdl.s.nop", + "rocdl.s.setprio", + "rocdl.s.sleep", + "rocdl.s.wait.asynccnt", + "rocdl.s.wait.dscnt", + "rocdl.s.wait.expcnt", + "rocdl.s.wait.loadcnt", + "rocdl.s.wait.storecnt", + "rocdl.s.wait.tensorcnt", + "rocdl.s.waitcnt", + "rocdl.s.wakeup.barrier", + "rocdl.sched.barrier", + "rocdl.sched.group.barrier", + "rocdl.sdot2", + "rocdl.sdot4", + "rocdl.sdot8", + "rocdl.sin", + "rocdl.smfmac.f32.16x16x128.bf8.bf8", + "rocdl.smfmac.f32.16x16x128.bf8.fp8", + "rocdl.smfmac.f32.16x16x128.fp8.bf8", + "rocdl.smfmac.f32.16x16x128.fp8.fp8", + "rocdl.smfmac.f32.16x16x32.bf16", + "rocdl.smfmac.f32.16x16x32.f16", + "rocdl.smfmac.f32.16x16x64.bf16", + "rocdl.smfmac.f32.16x16x64.bf8.bf8", + "rocdl.smfmac.f32.16x16x64.bf8.fp8", + "rocdl.smfmac.f32.16x16x64.f16", + "rocdl.smfmac.f32.16x16x64.fp8.bf8", + "rocdl.smfmac.f32.16x16x64.fp8.fp8", + "rocdl.smfmac.f32.32x32x16.bf16", + "rocdl.smfmac.f32.32x32x16.f16", + "rocdl.smfmac.f32.32x32x32.bf16", + "rocdl.smfmac.f32.32x32x32.bf8.bf8", + "rocdl.smfmac.f32.32x32x32.bf8.fp8", + "rocdl.smfmac.f32.32x32x32.f16", + "rocdl.smfmac.f32.32x32x32.fp8.bf8", + "rocdl.smfmac.f32.32x32x32.fp8.fp8", + "rocdl.smfmac.f32.32x32x64.bf8.bf8", + "rocdl.smfmac.f32.32x32x64.bf8.fp8", + "rocdl.smfmac.f32.32x32x64.fp8.bf8", + "rocdl.smfmac.f32.32x32x64.fp8.fp8", + "rocdl.smfmac.i32.16x16x128.i8", + "rocdl.smfmac.i32.16x16x64.i8", + "rocdl.smfmac.i32.32x32x32.i8", + "rocdl.smfmac.i32.32x32x64.i8", + "rocdl.sqrt", + "rocdl.sudot4", + "rocdl.sudot8", + "rocdl.swmmac.bf16.16x16x32.bf16", + "rocdl.swmmac.bf16.16x16x64.bf16", + "rocdl.swmmac.bf16f32.16x16x64.bf16", + "rocdl.swmmac.f16.16x16x128.bf8.bf8", + "rocdl.swmmac.f16.16x16x128.bf8.fp8", + "rocdl.swmmac.f16.16x16x128.fp8.bf8", + "rocdl.swmmac.f16.16x16x128.fp8.fp8", + "rocdl.swmmac.f16.16x16x32.f16", + "rocdl.swmmac.f16.16x16x64.f16", + "rocdl.swmmac.f32.16x16x128.bf8.bf8", + "rocdl.swmmac.f32.16x16x128.bf8.fp8", + "rocdl.swmmac.f32.16x16x128.fp8.bf8", + "rocdl.swmmac.f32.16x16x128.fp8.fp8", + "rocdl.swmmac.f32.16x16x32.bf16", + "rocdl.swmmac.f32.16x16x32.bf8.bf8", + "rocdl.swmmac.f32.16x16x32.bf8.fp8", + "rocdl.swmmac.f32.16x16x32.f16", + "rocdl.swmmac.f32.16x16x32.fp8.bf8", + "rocdl.swmmac.f32.16x16x32.fp8.fp8", + "rocdl.swmmac.f32.16x16x64.bf16", + "rocdl.swmmac.f32.16x16x64.f16", + "rocdl.swmmac.i32.16x16x128.iu8", + "rocdl.swmmac.i32.16x16x32.iu4", + "rocdl.swmmac.i32.16x16x32.iu8", + "rocdl.swmmac.i32.16x16x64.iu4", + "rocdl.tanh", + "rocdl.tensor.load.to.lds", + "rocdl.tensor.store.from.lds", + "rocdl.udot2", + "rocdl.udot4", + "rocdl.udot8", + "rocdl.update.dpp", + "rocdl.wait.asyncmark", + "rocdl.wave.barrier", + "rocdl.wave.id", + "rocdl.wavefrontsize", + "rocdl.wmma.bf16.16x16x16.bf16", + "rocdl.wmma.bf16.16x16x32.bf16", + "rocdl.wmma.bf16f32.16x16x32.bf16", + "rocdl.wmma.f16.16x16x128.bf8_bf8", + "rocdl.wmma.f16.16x16x128.bf8_fp8", + "rocdl.wmma.f16.16x16x128.fp8_bf8", + "rocdl.wmma.f16.16x16x128.fp8_fp8", + "rocdl.wmma.f16.16x16x16.f16", + "rocdl.wmma.f16.16x16x32.f16", + "rocdl.wmma.f16.16x16x64.bf8_bf8", + "rocdl.wmma.f16.16x16x64.bf8_fp8", + "rocdl.wmma.f16.16x16x64.fp8_bf8", + "rocdl.wmma.f16.16x16x64.fp8_fp8", + "rocdl.wmma.f32.16x16x128.bf8_bf8", + "rocdl.wmma.f32.16x16x128.bf8_fp8", + "rocdl.wmma.f32.16x16x128.fp8_bf8", + "rocdl.wmma.f32.16x16x128.fp8_fp8", + "rocdl.wmma.f32.16x16x16.bf16", + "rocdl.wmma.f32.16x16x16.bf8_bf8", + "rocdl.wmma.f32.16x16x16.bf8_fp8", + "rocdl.wmma.f32.16x16x16.f16", + "rocdl.wmma.f32.16x16x16.fp8_bf8", + "rocdl.wmma.f32.16x16x16.fp8_fp8", + "rocdl.wmma.f32.16x16x32.bf16", + "rocdl.wmma.f32.16x16x32.f16", + "rocdl.wmma.f32.16x16x4.f32", + "rocdl.wmma.f32.16x16x64.bf8_bf8", + "rocdl.wmma.f32.16x16x64.bf8_fp8", + "rocdl.wmma.f32.16x16x64.fp8_bf8", + "rocdl.wmma.f32.16x16x64.fp8_fp8", + "rocdl.wmma.i32.16x16x16.iu4", + "rocdl.wmma.i32.16x16x16.iu8", + "rocdl.wmma.i32.16x16x32.iu4", + "rocdl.wmma.i32.16x16x64.iu8", + "rocdl.wmma.scale.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale.f32.32x16x128.f4", + "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale16.f32.32x16x128.f4", + "rocdl.workgroup.id.x", + "rocdl.workgroup.id.y", + "rocdl.workgroup.id.z", + "rocdl.workitem.id.x", + "rocdl.workitem.id.y", + "rocdl.workitem.id.z", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectAttributeRoundTrip(ctx: *mlir.Context, attribute: *const mlir.Attribute) !void { + var buffer: [2048]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try attribute.format(&writer); + try std.testing.expect(attribute.eql(try mlir.Attribute.parse(ctx, writer.buffered()))); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(rocdl); + std.testing.refAllDecls(rocdl.ops); + std.testing.refAllDecls(rocdl.types); + std.testing.refAllDecls(rocdl.annotations); + inline for (rocdl.attributes.enum_attributes) |A| std.testing.refAllDecls(A); + std.testing.refAllDecls(rocdl.TargetAttr); +} + +test "packed FP8 conversion uses the registered LLVM ROCDL dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.cvt.pk.f32.fp8")); + + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + for ([_]bool{ false, true }) |high| { + const op = rocdl.cvtPkF32Fp8(ctx, block.argument(0), high, loc); + defer op.deinit(); + try std.testing.expect(op.verify()); + try std.testing.expect(op.result(0).type_().eql(.vector(&.{2}, .float(ctx, .f32)))); + try std.testing.expect(op.attributeByName("wordSel").?.eql(.boolean(ctx, high))); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectEqual(operation_names.len, @as(usize, @intCast(c.mlirROCDLGetOperationCount(ctx.ptr())))); + for (operation_names, 0..) |name, i| { + try std.testing.expect(ctx.isRegisteredOperation(name)); + for (operation_names[0..i]) |earlier| try std.testing.expect(!std.mem.eql(u8, name, earlier)); + } + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "all ROCDL enum attributes construct and round-trip every named value" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (rocdl.attributes.enum_attributes) |A| { + for (std.enums.values(A.Value)) |value| { + const attribute = try A.get(ctx, .{ .value = value }); + try std.testing.expect(attribute.isA(A) != null); + try std.testing.expectEqual(value, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); + } + } + const mask: rocdl.SchedGroupMask = .combine(&.{ .valu, .mfma_wmma, .ds_read }); + try std.testing.expect(mask.contains(.mfma_wmma)); + try std.testing.expect(!mask.contains(.all_vmem)); + const attribute = try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = mask }); + try std.testing.expectEqual(mask, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); +} + +test "ROCDL enum constructors reject invalid values and flag bits" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), 5).ptr == null); + try std.testing.expect(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), 65536).ptr == null); + try std.testing.expectError(error.InvalidMlir, rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = @fromBackingInt(1 << 20) })); + try std.testing.expectError(error.InvalidMlir, rocdl.PreGfx12CachePolicyAttr.get(ctx, .{ .value = @fromBackingInt(1 << 6) })); +} + +test "target attributes expose all fields and reject invalid options" { + const ctx = try testContext(); + defer ctx.deinit(); + const defaults = try rocdl.TargetAttr.get(ctx, .{}); + try std.testing.expectEqual(@as(i32, 2), defaults.optLevel()); + try std.testing.expectEqualStrings("amdgcn-amd-amdhsa", defaults.triple()); + try std.testing.expectEqualStrings("gfx900", defaults.chip()); + try std.testing.expectEqualStrings("", defaults.features()); + try std.testing.expectEqualStrings("600", defaults.abi()); + try std.testing.expect(defaults.flags() == null and defaults.link() == null); + try std.testing.expect(defaults.hasWave64()); + try std.testing.expect(!defaults.hasFastMath()); + try std.testing.expect(defaults.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, defaults.attribute()); + + const flags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(ctx, &.{ + .named(ctx, "no_wave64", .unit(ctx)), + .named(ctx, "fast", .unit(ctx)), + .named(ctx, "daz", .unit(ctx)), + .named(ctx, "finite_only", .unit(ctx)), + .named(ctx, "unsafe_math", .unit(ctx)), + .named(ctx, "unsafe_sqrt", .unit(ctx)), + })); + const link: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.string(ctx, "device.bc")})); + const target = try rocdl.TargetAttr.get(ctx, .{ .optLevel = 3, .chip = "gfx942", .features = "+xnack", .abi = "500", .flags = flags, .link = link }); + try std.testing.expectEqual(@as(i32, 3), target.optLevel()); + try std.testing.expectEqualStrings("gfx942", target.chip()); + try std.testing.expectEqualStrings("+xnack", target.features()); + try std.testing.expectEqualStrings("500", target.abi()); + try std.testing.expect(target.flags().?.eql(flags)); + try std.testing.expect(target.link().?.eql(link)); + try std.testing.expect(target.hasFlag("fast") and target.hasFastMath()); + try std.testing.expect(!target.hasWave64()); + try std.testing.expect(target.hasDaz() and target.hasFiniteOnly()); + try std.testing.expect(target.hasUnsafeMath() and !target.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, target.attribute()); + + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .optLevel = 4 })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .triple = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .chip = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .abi = "700" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = .unit(ctx) })); + const badLink: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.unit(ctx)})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .link = badLink })); + const other = try testContext(); + defer other.deinit(); + const otherFlags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(other, &.{})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = otherFlags })); +} + +test "ROCDL barriers buffer loads and matrix operations verify and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const module = try mlir.Module.parse(ctx, + \\module { + \\ llvm.func @test(%rsrc: vector<4xi32>, %offset: i32, %a: f32, %c: vector<4xf32>) { + \\ llvm.return + \\ } + \\} + ); + defer module.deinit(); + const function = module.body().firstOperation().?; + const block = function.region(0).firstBlock().?; + const terminator = block.terminator().?; + const i32Type: *const mlir.Type = .int(ctx, .i32); + const f32Type: *const mlir.Type = .float(ctx, .f32); + const vec4f32 = rocdl.types.v4f32(ctx); + const zero: *const mlir.Attribute = .int(ctx, .i32, 0); + const opList = .{ + rocdl.ops.s_barrier(ctx, loc), + rocdl.ops.s_sleep(ctx, .int(ctx, .i32, 1), loc), + rocdl.ops.workitem_id_x(ctx, i32Type, .{}, loc), + rocdl.ops.sched_barrier(ctx, (try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = .combine(&.{ .valu, .mfma_wmma }) })).attribute(), loc), + rocdl.ops.raw_buffer_load(ctx, block.argument(0), block.argument(1), block.argument(1), f32Type, .{}, loc), + rocdl.ops.raw_buffer_load( + ctx, + block.argument(0), + block.argument(1), + block.argument(1), + f32Type, + .{ .aux = (try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) })).attribute() }, + loc, + ), + rocdl.ops.mfma_f32_16x16x4f32(ctx, block.argument(2), block.argument(2), block.argument(3), vec4f32, zero, zero, (try rocdl.MFMAPermBAttr.get(ctx, .{ .value = .none })).attribute(), loc), + }; + inline for (opList) |op| { + try std.testing.expect(op.verify()); + block.insertOwnedOperationBefore(terminator, op); + } + rocdl.annotations.annotateFunction(ctx, function, .{ + .kernel = true, + .reqdWorkGroupSize = .{ 64, 1, 1 }, + .flatWorkGroupSize = "64,64", + .maxFlatWorkGroupSize = 64, + .wavesPerEu = .{ .min = 1, .max = 2 }, + .unsafeFpAtomics = true, + }); + rocdl.annotations.annotateMemory(ctx, opList[4], .{ .lastUse = true, .noRemoteMemory = true, .noFineGrainedMemory = true, .ignoreDenormalMode = true }); + try std.testing.expect(opList[4].attributeByName("aux").?.eql(.int(ctx, .i32, 0))); + try std.testing.expect(opList[5].attributeByName("aux").?.isA(rocdl.Gfx942CachePolicyAttr) != null); + try std.testing.expect(function.attributeByName("rocdl.waves_per_eu").?.eql(.string(ctx, "1,2"))); + try std.testing.expect(module.operation().verify()); + var buffer: [16384]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try module.format(&writer); + const parsed = try mlir.Module.parse(ctx, writer.buffered()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "ROCDL annotations preserve module flag defaults and explicit false" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = mlir.Module.init(.unknown(ctx)); + defer module.deinit(); + rocdl.annotations.annotateModule(ctx, module.operation(), .{}); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack") == null); + rocdl.annotations.annotateModule(ctx, module.operation(), .{ .xnack = false, .sramecc = true }); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack").?.eql(.boolean(ctx, false))); + try std.testing.expect(module.operation().attributeByName("rocdl.sramecc").?.eql(.boolean(ctx, true))); + try std.testing.expect(module.operation().verify()); + const pointer = rocdl.pointerType(ctx, .barrier); + try std.testing.expect(pointer.eql(try mlir.Type.parse(ctx, "!llvm.ptr<15>"))); + try std.testing.expect(rocdl.pointerType(ctx, .buffer_resource).eql(try mlir.Type.parse(ctx, "!llvm.ptr<8>"))); +} diff --git a/mlir/dialects/rocdl/types.zig b/mlir/dialects/rocdl/types.zig new file mode 100644 index 00000000..a91ede15 --- /dev/null +++ b/mlir/dialects/rocdl/types.zig @@ -0,0 +1,147 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); + +/// ROCDL address spaces use LLVM pointer types. +pub const AddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + private = 5, + buffer_fat_pointer = 7, + buffer_resource = 8, + buffer_strided_pointer = 9, + barrier = 15, +}; + +pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { + return llvm.pointerType(ctx, space); +} + +/// `vector<16xbf16>`, as used by ROCDL operations. +pub fn v16bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .bf16)); +} + +/// `vector<16xf16>`, as used by ROCDL operations. +pub fn v16f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f16)); +} + +/// `vector<16xf32>`, as used by ROCDL operations. +pub fn v16f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f32)); +} + +/// `vector<16xi32>`, as used by ROCDL operations. +pub fn v16i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .int(ctx, .i32)); +} + +/// `vector<2xbf16>`, as used by ROCDL operations. +pub fn v2bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .bf16)); +} + +/// `vector<2xf16>`, as used by ROCDL operations. +pub fn v2f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f16)); +} + +/// `vector<2xf32>`, as used by ROCDL operations. +pub fn v2f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f32)); +} + +/// `vector<2xi16>`, as used by ROCDL operations. +pub fn v2i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i16)); +} + +/// `vector<2xi32>`, as used by ROCDL operations. +pub fn v2i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i32)); +} + +/// `vector<32xbf16>`, as used by ROCDL operations. +pub fn v32bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .bf16)); +} + +/// `vector<32xf16>`, as used by ROCDL operations. +pub fn v32f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f16)); +} + +/// `vector<32xf32>`, as used by ROCDL operations. +pub fn v32f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f32)); +} + +/// `vector<32xi32>`, as used by ROCDL operations. +pub fn v32i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .int(ctx, .i32)); +} + +/// `vector<3xi32>`, as used by ROCDL operations. +pub fn v3i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{3}, .int(ctx, .i32)); +} + +/// `vector<4xf16>`, as used by ROCDL operations. +pub fn v4f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f16)); +} + +/// `vector<4xf32>`, as used by ROCDL operations. +pub fn v4f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f32)); +} + +/// `vector<4xf64>`, as used by ROCDL operations. +pub fn v4f64(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f64)); +} + +/// `vector<4xi16>`, as used by ROCDL operations. +pub fn v4i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i16)); +} + +/// `vector<4xi32>`, as used by ROCDL operations. +pub fn v4i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i32)); +} + +/// `vector<6xi32>`, as used by ROCDL operations. +pub fn v6i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{6}, .int(ctx, .i32)); +} + +/// `vector<8xbf16>`, as used by ROCDL operations. +pub fn v8bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .bf16)); +} + +/// `vector<8xf16>`, as used by ROCDL operations. +pub fn v8f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f16)); +} + +/// `vector<8xf32>`, as used by ROCDL operations. +pub fn v8f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f32)); +} + +/// `vector<8xi16>`, as used by ROCDL operations. +pub fn v8i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i16)); +} + +/// `vector<8xi32>`, as used by ROCDL operations. +pub fn v8i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i32)); +} diff --git a/mlir/dialects/scf/BUILD.bazel b/mlir/dialects/scf/BUILD.bazel new file mode 100644 index 00000000..1d7a0b3c --- /dev/null +++ b/mlir/dialects/scf/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "scf", + import_name = "mlir/dialects/scf", + main = "scf.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPISCF", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":scf", + "//mlir", + ], +) diff --git a/mlir/dialects/scf/scf.zig b/mlir/dialects/scf/scf.zig new file mode 100644 index 00000000..1c82a9e0 --- /dev/null +++ b/mlir/dialects/scf/scf.zig @@ -0,0 +1,363 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("scf"); +} + +pub const ForOpts = struct { + /// Per td: `scf.for` uses signed comparison by default. Setting this emits + /// the `unsignedCmp` UnitAttr so the induction variable advances with + /// unsigned comparison instead. + unsigned_cmp: bool = false, + num_stages: ?i32 = null, +}; + +/// scf.for — operands are (lower, upper, step, init_args...) as a flat list. +/// Result arity equals init_args arity; result types must match init_args types. +/// +/// The body block must have arguments `[iv_type, *init_arg_types]` and be +/// terminated with `scf.yield`. +pub fn for_( + ctx: *mlir.Context, + lower: *const mlir.Value, + upper: *const mlir.Value, + step: *const mlir.Value, + init_args: []const *const mlir.Value, + body: *mlir.Block, + opts: ForOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(&.{ lower, upper, step }); + operands.appendSliceAssumeCapacity(init_args); + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (init_args) |v| result_types.appendAssumeCapacity(v.type_()); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.unsigned_cmp) { + attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", .unit(ctx))); + } + if (opts.num_stages) |n| { + attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", .int(ctx, .i32, n))); + } + + return mlir.Operation.make(ctx, "scf.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // Per-op verify fails with "operand #0 does not dominate this use" when + // the scf.for references outer SSA values that only become reachable + // once it's appended to the parent block. Defer to module-level verify. + .verify = false, + .location = location, + }); +} + +/// scf.if — cond is i1; then/else blocks must be terminated with `scf.yield` +/// (or implicitly for no-result forms). +/// +/// Built via raw OperationState because it has two separate regions. +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.if", location); + state.addOperands(&.{cond}); + state.addResults(result_types); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.if"); +} + +/// scf.yield — variadic result terminator for scf.for / scf.if regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +/// scf.while — two regions. The `before` region computes the condition and +/// terminates with `scf.condition`. The `after` region runs when the condition +/// is true and terminates with `scf.yield`. The arguments of `before` have the +/// types of `inits`. The trailing operands of `scf.condition` feed both the +/// `after` region's arguments and the `scf.while` op's results. +pub fn while_( + ctx: *mlir.Context, + inits: []const *const mlir.Value, + result_types: []const *const mlir.Type, + before_block: *mlir.Block, + after_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.while", location); + state.addOperands(inits); + state.addResults(result_types); + + const before_region = mlir.Region.init(); + before_region.appendOwnedBlock(before_block); + const after_region = mlir.Region.init(); + after_region.appendOwnedBlock(after_block); + state.addOwnedRegions(&.{ before_region, after_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.while"); +} + +/// scf.condition — terminator of the `before` region of scf.while. +/// First operand is i1 (the continuation condition). The remaining operands +/// are forwarded to either the `after` region's arguments (if true) or the +/// scf.while's results (if false). +pub fn condition( + ctx: *mlir.Context, + cond: *const mlir.Value, + args: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + buf.appendAssumeCapacity(cond); + buf.appendSliceAssumeCapacity(args); + return mlir.Operation.make(ctx, "scf.condition", .{ + .operands = .{ .flat = buf.constSlice() }, + .verify = false, + .location = location, + }); +} + +pub const ExecuteRegionOpts = struct { + no_inline: bool = false, +}; + +/// scf.execute_region — runs `body` exactly once, returns the yielded values. +/// `body` has no arguments but may span multiple blocks (unlike for/if). +pub fn execute_region( + ctx: *mlir.Context, + result_types: []const *const mlir.Type, + body: *mlir.Block, + opts: ExecuteRegionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.no_inline) { + attrs.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); + } + return mlir.Operation.make(ctx, "scf.execute_region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // execute_region has its own region-verify; defer to the module-level + // verify so unresolved SSA uses don't block construction. + .verify = false, + .location = location, + }); +} + +/// scf.parallel — multi-dim parallel for loop with optional reductions. +/// Operands are (lbs, ubs, steps, inits). `body_block` takes one `index` arg +/// per loop dim and must be terminated with `scf.reduce` (or no-op if inits is +/// empty — the parser auto-inserts an empty reduce). +pub fn parallel( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + inits: []const *const mlir.Value, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(inits); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(inits.len), + }; + return mlir.Operation.make(ctx, "scf.parallel", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .verify = false, + .location = location, + }); +} + +/// scf.reduce — terminator of `scf.parallel`. Each region is a reduction body +/// taking two args (lhs, rhs) matching the corresponding operand's type, +/// terminated with `scf.reduce.return`. +pub fn reduce( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + reduction_blocks: []const *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + std.debug.assert(operands.len == reduction_blocks.len); + var state: mlir.OperationState = .init("scf.reduce", location); + state.addOperands(operands); + + var regions: stdx.BoundedArray(*mlir.Region, 16) = .empty; + for (reduction_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.reduce"); +} + +pub fn reduce_return(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.reduce.return", .{ + .operands = .{ .flat = &.{value} }, + .verify = false, + .location = location, + }); +} + +/// scf.forall — multi-dim parallel region, similar to `scf.parallel` but using +/// tensor-based `shared_outs` instead of init values. This wrapper builds the +/// fully-dynamic form (all lb/ub/step are runtime values). +/// +/// `body_block` must have `rank + shared_outs.len` arguments (one index per +/// loop dim, plus one per shared_out) and must be terminated with +/// `scf.forall.in_parallel`. +pub fn forall( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + shared_outs: []const *const mlir.Value, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + opts: struct { + mapping: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(shared_outs); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(shared_outs.len), + }; + + // Fully-dynamic form: the static arrays use the kDynamic sentinel + // (mlir::ShapedType::kDynamic == INT64_MIN). We build one slot per loop + // dim, all dynamic. This matches how MLIR printers emit + // `scf.forall (%iv) = (%lb) to (%ub) step (%step)` when no static + // constants are folded in. + const k_dynamic: i64 = std.math.minInt(i64); + var sentinel: stdx.BoundedArray(i64, 16) = .empty; + for (0..lbs.len) |_| sentinel.appendAssumeCapacity(k_dynamic); + const sentinel_slice = sentinel.constSlice(); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "staticLowerBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticUpperBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticStep", .denseArray(ctx, .i64, sentinel_slice)), + }); + if (opts.mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); + + return mlir.Operation.make(ctx, "scf.forall", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// scf.forall.in_parallel — terminator for scf.forall. The inner region is +/// a single block containing a flat list of ops (typically +/// tensor.parallel_insert_slice). `body_block` must be built by the caller. +pub fn forall_in_parallel( + ctx: *mlir.Context, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.forall.in_parallel", location); + const region = mlir.Region.init(); + region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create scf.forall.in_parallel"); +} + +/// scf.index_switch — branches to case_blocks[i] if arg == cases[i], else +/// falls through to default_block. Blocks must be terminated with scf.yield. +/// The op's results have the types yielded by each region. +pub fn index_switch( + ctx: *mlir.Context, + arg: *const mlir.Value, + cases: []const i64, + default_block: *mlir.Block, + case_blocks: []const *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(cases.len == case_blocks.len); + var state: mlir.OperationState = .init("scf.index_switch", location); + state.addOperands(&.{arg}); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "cases", .denseArray(ctx, .i64, cases)), + }); + + // First region is the default, then one per case — this is how + // `VariadicRegion>:$caseRegions` is positioned in the td. + const default_region = mlir.Region.init(); + default_region.appendOwnedBlock(default_block); + + var regions: stdx.BoundedArray(*mlir.Region, 64) = .empty; + regions.appendAssumeCapacity(default_region); + for (case_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.index_switch"); +} diff --git a/mlir/dialects/scf/test.zig b/mlir/dialects/scf/test.zig new file mode 100644 index 00000000..dceac66e --- /dev/null +++ b/mlir/dialects/scf/test.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +const operation_names: []const []const u8 = &.{ + "scf.condition", + "scf.execute_region", + "scf.for", + "scf.forall", + "scf.forall.in_parallel", + "scf.if", + "scf.index_switch", + "scf.parallel", + "scf.reduce", + "scf.reduce.return", + "scf.while", + "scf.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(scf); + inline for (comptime std.meta.declarations(scf)) |decl| { + const T = @field(scf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + scf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/shardy/BUILD.bazel b/mlir/dialects/shardy/BUILD.bazel new file mode 100644 index 00000000..4077f902 --- /dev/null +++ b/mlir/dialects/shardy/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "shardy", + import_name = "mlir/dialects/shardy", + main = "shardy.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@shardy//shardy/integrations/c:sdy_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":shardy", + "//mlir", + "//mlir/dialects/func", + ], +) diff --git a/mlir/dialects/shardy/shardy.zig b/mlir/dialects/shardy/shardy.zig new file mode 100644 index 00000000..d73bf81b --- /dev/null +++ b/mlir/dialects/shardy/shardy.zig @@ -0,0 +1,364 @@ +//! Bindings for OpenXLA shardy dialect +//! +//! - dialect overview: https://openxla.org/shardy/sdy_dialect +//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("sdy"); +} + +/// `sdy.sharding`, the sharding of a function argument or result. +pub fn shardingNamedAttr(ctx: *mlir.Context, sharding: *const TensorShardingAttr) mlir.NamedAttribute { + return .named(ctx, "sdy.sharding", sharding.attribute()); +} + +/// `sdy.mesh`. Not verified: it only verifies inside a module. +pub fn mesh(ctx: *mlir.Context, sym_name: []const u8, mesh_: *const MeshAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.mesh", .{ + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, sym_name)), + .named(ctx, "mesh", mesh_.attribute()), + }, + .verify = false, + .location = location, + }); +} + +pub fn sharding_constraint(ctx: *mlir.Context, input: *const mlir.Value, sharding: *const TensorShardingAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.sharding_constraint", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "sharding", sharding.attribute()), + }, + .location = location, + }); +} + +pub fn manual_computation( + ctx: *mlir.Context, + tensors: []const *const mlir.Value, + result_types: []const *const mlir.Type, + body_block: *mlir.Block, + in_shardings: *const TensorShardingPerValueAttr, + out_shardings: *const TensorShardingPerValueAttr, + manual_axes: *const ManualAxesAttr, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.manual_computation", .{ + .operands = .{ .flat = tensors }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "in_shardings", in_shardings.attribute()), + .named(ctx, "out_shardings", out_shardings.attribute()), + .named(ctx, "manual_axes", manual_axes.attribute()), + }, + .location = location, + }); +} + +/// `sdy.return`. Not verified: a terminator only verifies inside its parent. +pub fn return_(ctx: *mlir.Context, results: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.return", .{ + .operands = .{ .flat = results }, + .verify = false, + .location = location, + }); +} + +pub const MeshAxisAttr = opaque { + const M = mlir.Methods(MeshAxisAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + name: []const u8, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAxisAttr { + return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(args.name), args.size).ptr); + } + + pub fn attribute(self: *const MeshAxisAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const MeshAxisAttr) []const u8 { + return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); + } + + pub fn size(self: *const MeshAxisAttr) i64 { + return c.sdyMeshAxisAttrGetSize(self.ptr()); + } +}; + +pub const MeshAttr = opaque { + const M = mlir.Methods(MeshAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const MeshAxisAttr, + device_ids: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAttr { + const axes: []const c.struct_MlirAttribute = @ptrCast(args.axes); + return @ptrCast(c.sdyMeshAttrGet( + ctx.ptr(), + @intCast(axes.len), + axes.ptr, + @intCast(args.device_ids.len), + args.device_ids.ptr, + ).ptr); + } + + pub fn attribute(self: *const MeshAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SubAxisInfoAttr = opaque { + const M = mlir.Methods(SubAxisInfoAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + pre_size: i64, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const SubAxisInfoAttr { + return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), args.pre_size, args.size).ptr); + } + + pub fn attribute(self: *const SubAxisInfoAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn preSize(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); + } + + pub fn size(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetSize(self.ptr()); + } +}; + +pub const AxisRefAttr = opaque { + const M = mlir.Methods(AxisRefAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttr { + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); + } + + pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttr { + const sub_axis: *const SubAxisInfoAttr = .get(ctx, .{ .pre_size = pre_size, .size = size }); + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis.ptr()).ptr); + } + + pub fn attribute(self: *const AxisRefAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const AxisRefAttr) []const u8 { + return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); + } + + pub fn subAxisInfo(self: *const AxisRefAttr) ?*const SubAxisInfoAttr { + const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); + return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); + } +}; + +pub const DimensionShardingAttr = opaque { + const M = mlir.Methods(DimensionShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const AxisRefAttr, + is_closed: bool, + priority: ?i64 = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DimensionShardingAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(args.axes); + return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, args.is_closed, args.priority orelse -1).ptr); + } + + pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = true }); + } + + pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = false }); + } + + pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttr { + return closed(ctx, &.{}); + } + + pub fn attribute(self: *const DimensionShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const DimensionShardingAttr) usize { + return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const DimensionShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn isClosed(self: *const DimensionShardingAttr) bool { + return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); + } + + pub fn priority(self: *const DimensionShardingAttr) ?i64 { + const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); + return if (value == -1) null else value; + } +}; + +pub const TensorShardingAttr = opaque { + const M = mlir.Methods(TensorShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + mesh: []const u8, + dimensions: []const *const DimensionShardingAttr, + replicated_axes: []const *const AxisRefAttr = &.{}, + unreduced_axes: []const *const AxisRefAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorShardingAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); + const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); + const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); + const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); + return @ptrCast(c.sdyTensorShardingAttrGet( + ctx.ptr(), + mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), + @intCast(dimensions.len), + dimensions.ptr, + @intCast(replicated.len), + replicated.ptr, + @intCast(unreduced.len), + unreduced.ptr, + 0, + ).ptr); + } + + pub fn attribute(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn meshOrRef(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); + } + + pub fn numDimensions(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); + } + + pub fn dimension(self: *const TensorShardingAttr, index: usize) *const DimensionShardingAttr { + return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numReplicatedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); + } + + pub fn replicatedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numUnreducedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); + } + + pub fn unreducedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); + } +}; + +pub const TensorShardingPerValueAttr = opaque { + const M = mlir.Methods(TensorShardingPerValueAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const TensorShardingPerValueAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); + return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const TensorShardingPerValueAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ManualAxesAttr = opaque { + const M = mlir.Methods(ManualAxesAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAManualAxesAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); + const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); + return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const ManualAxesAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const ManualAxesAttr) usize { + return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const ManualAxesAttr, index: usize) []const u8 { + return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); + } +}; diff --git a/mlir/dialects/shardy/test.zig b/mlir/dialects/shardy/test.zig new file mode 100644 index 00000000..a4a757d2 --- /dev/null +++ b/mlir/dialects/shardy/test.zig @@ -0,0 +1,200 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); + +const ManualAxesAttr = shardy.ManualAxesAttr; +const TensorShardingAttr = shardy.TensorShardingAttr; +const TensorShardingPerValueAttr = shardy.TensorShardingPerValueAttr; + +const operation_names: []const []const u8 = &.{ + "sdy.manual_computation", + "sdy.mesh", + "sdy.return", + "sdy.sharding_constraint", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(shardy); + inline for (comptime std.meta.declarations(shardy)) |decl| { + const T = @field(shardy, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test TensorShardingAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "suggested_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_z")}), + .closed(ctx, &.{.named(ctx, "link_x")}), + }, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "fold_mesh", + .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "strategy_fold", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "mix_mesh", + .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "3d_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_x")}), + .closed(ctx, &.{.named(ctx, "link_y")}), + .closed(ctx, &.{.named(ctx, "link_z")}), + }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); + } +} + +test TensorShardingPerValueAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const shardings: [2]*const TensorShardingAttr = .{ + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + }), + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }), + }; + + const per_value = TensorShardingPerValueAttr.get(ctx, &shardings); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try per_value.attribute().format(&w.writer); + try std.testing.expectEqualSlices( + u8, + "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", + w.written(), + ); +} + +test ManualAxesAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const axes: [2]*const mlir.StringAttribute = .{ + .init(ctx, "link_x"), + .init(ctx, "link_y"), + }; + + const manual_axes: *const ManualAxesAttr = .get(ctx, &axes); + + try std.testing.expectEqual(2, manual_axes.numAxes()); + try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); + try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try manual_axes.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy", w.written()); +} + +test "sub-axis references build" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const ref = shardy.AxisRefAttr.subAxis(ctx, "x", 2, 4); + try std.testing.expectEqualStrings("x", ref.name()); + try std.testing.expectEqual(@as(i64, 4), ref.subAxisInfo().?.size()); +} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index 45e69239..d7214000 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -4,10 +4,10 @@ zig_library( name = "stablehlo", import_name = "mlir/dialects/stablehlo", main = "stablehlo.zig", - visibility = ["//mlir/dialects:__subpackages__"], + tags = ["manual"], + visibility = ["//visibility:public"], deps = [ "//mlir", - "//mlir:c", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], @@ -15,5 +15,9 @@ zig_library( zig_test( name = "test", - deps = [":stablehlo"], + main = "test.zig", + deps = [ + ":stablehlo", + "//mlir", + ], ) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index a21eec76..385a95b0 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -5,6 +5,10 @@ const mlir = @import("mlir"); const stringRef = mlir.stringRef; const stdx = @import("stdx"); +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("stablehlo"); +} + pub const abs = unaryOp("stablehlo.abs").call; pub const cosine = unaryOp("stablehlo.cosine").call; pub const sine = unaryOp("stablehlo.sine").call; @@ -93,7 +97,7 @@ pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, locat .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(lower))), }, .location = location, }); @@ -107,13 +111,41 @@ pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Valu }); } +pub const CompositeOpts = struct { + name: []const u8, + decomposition: []const u8, + composite_attributes: []const mlir.NamedAttribute = &.{}, + version: i32 = 0, +}; + +pub fn composite( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + opts: CompositeOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.composite", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "name", .string(ctx, opts.name)), + .named(ctx, "decomposition", .flatSymbolRef(ctx, opts.decomposition)), + .named(ctx, "composite_attributes", .dict(ctx, opts.composite_attributes)), + .named(ctx, "version", .int(ctx, .i32, opts.version)), + }, + .verify = false, + .location = location, + }); +} + pub const DotPrecision = union(enum) { fast, high, highest, algorithm: DotAlgorithm, - pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { + pub fn precision(self: DotPrecision) PrecisionAttr.Precision { return switch (self) { .fast => .DEFAULT, .high => .HIGH, @@ -123,9 +155,9 @@ pub const DotPrecision = union(enum) { }; } - pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, element_type: *const mlir.Type) ?*const mlir.Attribute { return switch (self) { - .algorithm => |algo| algo.asAttr(ctx, operand_type), + .algorithm => |algo| dotAlgorithmAttr(ctx, algo, element_type), else => null, }; } @@ -149,12 +181,12 @@ pub const DotAlgorithm = struct { }; }; -pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { +pub fn dotAlgorithmAttr(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { return @ptrCast(c.stablehloDotAlgorithmGet( ctx.ptr(), element_type.ptr(), element_type.ptr(), - mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), + mlir.Type.float(ctx, dot_algorithm.accumulation).ptr(), dot_algorithm.component_count, dot_algorithm.component_count, dot_algorithm.num_primitive_operations, @@ -178,22 +210,22 @@ pub fn dot_general( }, location: *const mlir.Location, ) *mlir.Operation { - const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); + const precisions: [2]*const mlir.Attribute = @splat(precisionAttr(ctx, opts.dot_precision.precision())); const attributes = [3]mlir.NamedAttribute{ .named( ctx, "dot_dimension_numbers", - dotDimensionNumbersAttribute(ctx, .{ + dotDimensionNumbersAttr(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, }), ), - .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), + .named(ctx, "precision_config", .array(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. .named(ctx, "algorithm", switch (opts.dot_precision) { - .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), + .algorithm => |v| dotAlgorithmAttr(ctx, v, lhs.type_()), else => undefined, }), }; @@ -215,9 +247,9 @@ pub fn constant( ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ .operands = .{ .flat = &.{} }, - .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, + .results = .{ .flat = &.{.rankedTensor(dims, elem_type)} }, .attributes = &.{ - .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type, null).shaped(), raw_bytes)), + .named(ctx, "value", .denseElements(.rankedTensor(dims, elem_type), raw_bytes)), }, .location = location, }); @@ -234,7 +266,7 @@ pub fn broadcast_in_dim( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), + .named(ctx, "broadcast_dimensions", .denseArray(ctx, .i64, dims)), }, .location = location, }); @@ -251,7 +283,7 @@ pub fn transpose( .operands = .{ .flat = &.{value} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), + .named(ctx, "permutation", .denseArray(ctx, .i64, opts.permutation)), }, .location = location, }); @@ -270,9 +302,9 @@ pub fn slice( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), - .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), - .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), + .named(ctx, "start_indices", .denseArray(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", .denseArray(ctx, .i64, limit_indices)), + .named(ctx, "strides", .denseArray(ctx, .i64, strides)), }, .location = location, }); @@ -283,7 +315,7 @@ pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimens .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "dimension", .int(ctx, .i64, dimension)), }, .location = location, }); @@ -326,7 +358,7 @@ pub fn gather( .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttr( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -335,8 +367,8 @@ pub fn gather( args.start_index_map, args.index_vector_dim, )), - .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), - .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), }, .location = location, }, @@ -353,7 +385,7 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + pub fn scatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { return @ptrCast(c.stablehloScatterDimensionNumbersGet( ctx.ptr(), @intCast(self.update_window_dims.len), @@ -387,9 +419,9 @@ pub fn scatter( .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), - .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), - .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), + .named(ctx, "scatter_dimension_numbers", args.scatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", .boolean(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -402,7 +434,7 @@ pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, l .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), + .named(ctx, "iota_dimension", .int(ctx, .i64, dimension)), }, .location = location, }); @@ -413,7 +445,7 @@ pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []con .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), }, .location = location, }); @@ -423,16 +455,16 @@ pub fn compare( ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, - comparison_direction: ComparisonDirection.Direction, - compare_type: CompareType.Type, + comparison_direction: ComparisonDirectionAttr.Direction, + compare_type: CompareTypeAttr.Type, location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), - .named(ctx, "compare_type", compareType(ctx, compare_type)), + .named(ctx, "comparison_direction", comparisonDirectionAttr(ctx, comparison_direction)), + .named(ctx, "compare_type", compareTypeAttr(ctx, compare_type)), }, .location = location, }); @@ -449,7 +481,7 @@ pub fn dynamic_slice( .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, new_dims)), }, .location = location, }); @@ -461,14 +493,14 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), - .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), - .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), + .named(ctx, "edge_padding_low", .denseArray(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", .denseArray(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", .denseArray(ctx, .i64, opts.interior)), }, .location = location, }); @@ -478,7 +510,7 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: TransposeAttribute.Type, + transpose_a: TransposeAttr.Type, }; pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { @@ -486,17 +518,17 @@ pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *co .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), - .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), - .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), - .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), + .named(ctx, "left_side", .int(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", .int(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttr(ctx, opts.transpose_a)), }, .location = location, }); } pub const FftOpts = struct { - kind: FftType.Type, + kind: FftTypeAttr.Type, length: []const i64, }; @@ -505,15 +537,15 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "fft_type", fftType(ctx, opts.kind)), - .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), + .named(ctx, "fft_type", fftTypeAttr(ctx, opts.kind)), + .named(ctx, "fft_length", .denseArray(ctx, .i64, opts.length)), }, .location = location, }); } -pub const RngDistribution = opaque { - const M = mlir.Methods(RngDistribution, c.MlirAttribute); +pub const RngDistributionAttr = opaque { + const M = mlir.Methods(RngDistributionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; pub const ptr = M.ptr; @@ -525,24 +557,32 @@ pub const RngDistribution = opaque { NORMAL, }; - pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { - return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngDistributionAttr { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn value(self: *const RngDistribution) Type { + pub fn value(self: *const RngDistributionAttr) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), ) orelse unreachable; } + + pub fn attribute(self: *const RngDistributionAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { - return @ptrCast(RngDistribution.get(ctx, value)); +pub fn rngDistributionAttr(ctx: *mlir.Context, value: RngDistributionAttr.Type) *const mlir.Attribute { + return RngDistributionAttr.get(ctx, .{ .value = value }).attribute(); } -pub const RngAlgorithm = opaque { - const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); +pub const RngAlgorithmAttr = opaque { + const M = mlir.Methods(RngAlgorithmAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; pub const ptr = M.ptr; @@ -555,28 +595,36 @@ pub const RngAlgorithm = opaque { PHILOX, }; - pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { - return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngAlgorithmAttr { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn value(self: *const RngAlgorithm) Type { + pub fn value(self: *const RngAlgorithmAttr) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), ) orelse unreachable; } + + pub fn attribute(self: *const RngAlgorithmAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { - return @ptrCast(RngAlgorithm.get(ctx, value)); +pub fn rngAlgorithmAttr(ctx: *mlir.Context, value: RngAlgorithmAttr.Type) *const mlir.Attribute { + return RngAlgorithmAttr.get(ctx, .{ .value = value }).attribute(); } -pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistributionAttr.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), + .named(ctx, "rng_distribution", rngDistributionAttr(ctx, rng_distribution)), }, .location = location, }); @@ -584,7 +632,7 @@ pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape pub fn rng_bit_generator( ctx: *mlir.Context, - rng_algorithm: RngAlgorithm.Type, + rng_algorithm: RngAlgorithmAttr.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, @@ -594,7 +642,7 @@ pub fn rng_bit_generator( .operands = .{ .flat = &.{initial_state} }, .results = .{ .flat = &.{ res_state_type, res_type } }, .attributes = &.{ - .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), + .named(ctx, "rng_algorithm", rngAlgorithmAttr(ctx, rng_algorithm)), }, .location = location, }); @@ -605,8 +653,8 @@ pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_b .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), - .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), + .named(ctx, "exponent_bits", .int(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", .int(ctx, .i32, mantissa_bits)), }, .location = location, }); @@ -620,12 +668,126 @@ pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, upda }); } +pub fn while_( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + cond_block: *mlir.Block, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state = mlir.OperationState.init("stablehlo.while", location); + state.addOperands(operands); + state.addResults(result_types); + + const cond_region = mlir.Region.init(); + cond_region.appendOwnedBlock(cond_block); + const body_region = mlir.Region.init(); + body_region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{ cond_region, body_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create stablehlo.while operation"); +} + +/// `stablehlo.if`. Not verified: its branches may use values defined +/// outside them, which only verify once the operation is in place. +pub fn if_( + ctx: *mlir.Context, + pred: *const mlir.Value, + true_branch_block: *mlir.Block, + false_branch_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.if", .{ + .operands = .{ .flat = &.{pred} }, + .result_type_inference = true, + .blocks = &.{ true_branch_block, false_branch_block }, + .verify = false, + .location = location, + }); +} + +pub fn reduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + dimensions: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), + }, + .location = location, + }); +} + +pub fn reduce_window( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + window_dimensions: []const i64, + opts: struct { + window_strides: ?[]const i64 = null, + base_dilations: ?[]const i64 = null, + window_dilations: ?[]const i64 = null, + /// (low, high) padding of each dimension. + padding: ?[]const [2]i64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "window_dimensions", .denseArray(ctx, .i64, window_dimensions))); + if (opts.window_strides) |v| attrs.appendAssumeCapacity(.named(ctx, "window_strides", .denseArray(ctx, .i64, v))); + if (opts.base_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "base_dilations", .denseArray(ctx, .i64, v))); + if (opts.window_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "window_dilations", .denseArray(ctx, .i64, v))); + if (opts.padding) |v| attrs.appendAssumeCapacity(.named(ctx, "padding", .denseElements( + .rankedTensor(&.{ @intCast(v.len), 2 }, .int(ctx, .i64)), + @as([]const i64, @ptrCast(v)), + ))); + return mlir.Operation.make(ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn sort( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + comparator_block: *mlir.Block, + opts: struct { + dimension: i64 = -1, + is_stable: bool = false, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{inputs} }, + .result_type_inference = true, + .blocks = &.{comparator_block}, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i64, opts.dimension)), + .named(ctx, "is_stable", .boolean(ctx, opts.is_stable)), + }, + .location = location, + }); +} + pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), + .named(ctx, "index", .int(ctx, .i32, index)), }, .location = location, }); @@ -649,7 +811,7 @@ pub const ConvolutionOpts = struct { output_spatial_dimensions: []const i64, feature_group_count: i64, batch_group_count: i64, - precision_config: []const PrecisionAttribute.Precision = &.{}, + precision_config: []const PrecisionAttr.Precision = &.{}, }; pub fn convolution( @@ -662,7 +824,7 @@ pub fn convolution( ) *mlir.Operation { var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = precisionAttribute(ctx, p); + max_precisions[i] = precisionAttr(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { @@ -672,12 +834,12 @@ pub fn convolution( .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), - .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64), null).shaped(), opts.pad_value)), - .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), - .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), - .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), - .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ + .named(ctx, "window_strides", .denseArray(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", .denseElements(.rankedTensor(opts.pad_shape, .int(ctx, .i64)), opts.pad_value)), + .named(ctx, "lhs_dilation", .denseArray(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", .denseArray(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", .denseArray(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttr(ctx, .{ .input_batch_dimension = opts.input_batch_dimension, .input_feature_dimension = opts.input_feature_dimension, .input_spatial_dimensions = opts.input_spatial_dimensions, @@ -688,15 +850,18 @@ pub fn convolution( .output_feature_dimension = opts.output_feature_dimension, .output_spatial_dimensions = opts.output_spatial_dimensions, })), - .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), - .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), - .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), + .named(ctx, "feature_group_count", .int(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", .int(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", .array(ctx, &max_precisions)), }, .location = location, }); } pub const CustomCallOpts = struct { + pub const MAX_OPERANDS = 64; + pub const MAX_RESULTS = 16; + pub const ApiVersion = enum(i32) { original = 1, status_returning = 2, @@ -711,19 +876,24 @@ pub const CustomCallOpts = struct { typed_ffi: *const mlir.Attribute, }; + pub const OutputOperandAlias = struct { + output_index: i64, + operand_index: i64, + }; + call_target_name: []const u8, has_side_effect: ?bool = null, api_version: ?ApiVersion = null, backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: ?[]const i64 = null, + output_operand_aliases: ?[]const OutputOperandAlias = null, additional_attributes: []const mlir.NamedAttribute = &.{}, }; pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { - const MAX_OPERANDS = 64; - const MAX_RESULTS = 16; + const MAX_OPERANDS = CustomCallOpts.MAX_OPERANDS; + const MAX_RESULTS = CustomCallOpts.MAX_RESULTS; const MINOR_TO_MAJOR = comptime blk: { var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; for (0..ret.len) |i| { @@ -732,56 +902,52 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result break :blk ret; }; - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .empty; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), + .named(ctx, "call_target_name", .string(ctx, opts.call_target_name)), }); attrs.appendSliceAssumeCapacity(opts.additional_attributes); if (opts.has_side_effect) |has_side_effects| { attrs.appendAssumeCapacity( - .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), + .named(ctx, "has_side_effect", .boolean(ctx, has_side_effects)), ); } const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), + .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), ); } if (opts.backend_config) |backend_config| { attrs.appendAssumeCapacity( .named(ctx, "backend_config", switch (backend_config) { .typed_ffi => |v| @ptrCast(v), - inline else => |v| mlir.stringAttribute(ctx, v), + inline else => |v| .string(ctx, v), }), ); } if (opts.output_operand_aliases) |output_operand_aliases| { - var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; - for (output_operand_aliases, 0..) |alias, output_index| { - const output_tuple_indices = if (result_types.len > 1) &[1]i64{@intCast(output_index)} else &.{}; + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .empty; + for (output_operand_aliases) |alias| { + const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; buffer.appendAssumeCapacity( - outputOperandAliasAttribute(ctx, .{ .operand_index = alias, .output_tuple_indices = output_tuple_indices }), + outputOperandAliasAttr(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), ); } attrs.appendAssumeCapacity( - .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), + .named(ctx, "output_operand_aliases", .array(ctx, buffer.constSlice())), ); } - var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .empty; if (opts.operand_layouts) |layouts| { for (layouts) |ol| { layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(ol.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), ol, ), ); @@ -791,31 +957,23 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = input.type_().isA(mlir.ShapedType).?; const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(ol.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), ol, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + .named(ctx, "operand_layouts", .array(ctx, layouts_buffer.constSlice())), ); layouts_buffer.clear(); if (opts.result_layouts) |layouts| { for (layouts) |rl| { layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(rl.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), rl, ), ); @@ -825,19 +983,15 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = result_type.isA(mlir.ShapedType).?; const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - mlir.denseElementsAttribute( - mlir.RankedTensorType.get( - &.{@intCast(rl.len)}, - mlir.indexType(ctx), - null, - ).shaped(), + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), rl, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), + .named(ctx, "result_layouts", .array(ctx, layouts_buffer.constSlice())), ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ @@ -848,6 +1002,14 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result }); } +pub fn optimizationBarrier(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return custom_call(ctx, &.{}, &.{value}, .{ .call_target_name = "CreateBuffer", @@ -856,28 +1018,40 @@ pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *cons } pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ - .call_target_name = "Pin", - .api_version = .typed_ffi, - }, location); + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, + location, + ); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ - .call_target_name = "Unpin", - .api_version = .typed_ffi, - }, location); + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, + location, + ); } pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ - .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, + .results = .{ .flat = &.{.rankedTensor(&.{}, .int(ctx, .u32))} }, .location = location, }); } -pub const DotDimensionNumbersAttribute = opaque { - const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); +pub const DotDimensionNumbersAttr = opaque { + const M = mlir.Methods(DotDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; pub const ptr = M.ptr; @@ -891,7 +1065,7 @@ pub const DotDimensionNumbersAttribute = opaque { rhs_contracting_dimensions: []const i64, }; - pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttr { return @ptrCast(c.stablehloDotDimensionNumbersGet( ctx.ptr(), @intCast(args.lhs_batching_dimensions.len), @@ -905,52 +1079,56 @@ pub const DotDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn lhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn lhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn rhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn rhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn lhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn lhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { + pub fn rhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { + pub fn rhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } + + pub fn attribute(self: *const DotDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); +pub fn dotDimensionNumbersAttr(ctx: *mlir.Context, args: DotDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttr.get(ctx, args)); } -pub const GatherDimensionNumbersAttribute = opaque { - const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); +pub const GatherDimensionNumbersAttr = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.eql(c.mlirAttributePrint); + pub const format = M.format(c.mlirAttributePrint); - pub fn init( + pub fn get( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -958,7 +1136,7 @@ pub const GatherDimensionNumbersAttribute = opaque { start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) *const GatherDimensionNumbersAttribute { + ) *const GatherDimensionNumbersAttr { return @ptrCast(c.stablehloGatherDimensionNumbersGet( ctx.ptr(), @intCast(offset_dims.len), @@ -975,52 +1153,52 @@ pub const GatherDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn offsetDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn offsetDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn collapsedSliceDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn collapsedSliceDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn startIndexMapSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn operandBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn operandBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { + pub fn startIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn startIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { + pub fn startIndexMapElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { + pub fn indexVectorDim(self: *const GatherDimensionNumbersAttr) usize { return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub fn gatherDimensionNumbersAttribute( +pub fn gatherDimensionNumbersAttr( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -1029,7 +1207,7 @@ pub fn gatherDimensionNumbersAttribute( start_index_map: []const i64, index_vector_dim: i64, ) *const mlir.Attribute { - return @ptrCast(GatherDimensionNumbersAttribute.init( + return @ptrCast(GatherDimensionNumbersAttr.get( ctx, offset_dims, collapsed_slice_dims, @@ -1040,8 +1218,8 @@ pub fn gatherDimensionNumbersAttribute( )); } -pub const ConvDimensionNumbersAttribute = opaque { - const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); +pub const ConvDimensionNumbersAttr = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; pub const ptr = M.ptr; @@ -1060,7 +1238,7 @@ pub const ConvDimensionNumbersAttribute = opaque { output_spatial_dimensions: []const i64, }; - pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttr { return @ptrCast(c.stablehloConvDimensionNumbersGet( ctx.ptr(), args.input_batch_dimension, @@ -1078,61 +1256,65 @@ pub const ConvDimensionNumbersAttribute = opaque { ).ptr); } - pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn inputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn inputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn inputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn inputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn kernelInputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn kernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn kernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn kernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn outputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { + pub fn outputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { + pub fn outputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { + pub fn outputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } + + pub fn attribute(self: *const ConvDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); +pub fn convDimensionNumbersAttr(ctx: *mlir.Context, args: ConvDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttr.get(ctx, args)); } -pub const OutputOperandAliasAttribute = opaque { - const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); +pub const OutputOperandAliasAttr = opaque { + const M = mlir.Methods(OutputOperandAliasAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; pub const ptr = M.ptr; @@ -1145,7 +1327,7 @@ pub const OutputOperandAliasAttribute = opaque { operand_tuple_indices: []const i64 = &.{}, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttr { return @ptrCast(c.stablehloOutputOperandAliasGet( ctx.ptr(), @intCast(args.output_tuple_indices.len), @@ -1155,14 +1337,18 @@ pub const OutputOperandAliasAttribute = opaque { @ptrCast(args.operand_tuple_indices), ).ptr); } + + pub fn attribute(self: *const OutputOperandAliasAttr) *const mlir.Attribute { + return @ptrCast(self); + } }; -pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { - return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); +pub fn outputOperandAliasAttr(ctx: *mlir.Context, args: OutputOperandAliasAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttr.get(ctx, args)); } -pub const PrecisionAttribute = opaque { - const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); +pub const PrecisionAttr = opaque { + const M = mlir.Methods(PrecisionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; pub const ptr = M.ptr; @@ -1175,22 +1361,30 @@ pub const PrecisionAttribute = opaque { HIGHEST, }; - pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { - return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Precision, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PrecisionAttr { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const PrecisionAttribute) Precision { - const value = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Precision, value) orelse unreachable; + pub fn value(self: *const PrecisionAttr) Precision { + const name = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Precision, name) orelse unreachable; + } + + pub fn attribute(self: *const PrecisionAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { - return @ptrCast(PrecisionAttribute.init(ctx, precision)); +pub fn precisionAttr(ctx: *mlir.Context, precision: PrecisionAttr.Precision) *const mlir.Attribute { + return PrecisionAttr.get(ctx, .{ .value = precision }).attribute(); } -pub const ComparisonDirection = opaque { - const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); +pub const ComparisonDirectionAttr = opaque { + const M = mlir.Methods(ComparisonDirectionAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; pub const ptr = M.ptr; @@ -1206,22 +1400,30 @@ pub const ComparisonDirection = opaque { LT, }; - pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { - return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Direction, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ComparisonDirectionAttr { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const ComparisonDirection) Direction { - const value = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Direction, value) orelse unreachable; + pub fn value(self: *const ComparisonDirectionAttr) Direction { + const name = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Direction, name) orelse unreachable; + } + + pub fn attribute(self: *const ComparisonDirectionAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { - return @ptrCast(ComparisonDirection.init(ctx, value)); +pub fn comparisonDirectionAttr(ctx: *mlir.Context, value: ComparisonDirectionAttr.Direction) *const mlir.Attribute { + return ComparisonDirectionAttr.get(ctx, .{ .value = value }).attribute(); } -pub const CompareType = opaque { - const M = mlir.Methods(CompareType, c.MlirAttribute); +pub const CompareTypeAttr = opaque { + const M = mlir.Methods(CompareTypeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; pub const ptr = M.ptr; @@ -1235,22 +1437,30 @@ pub const CompareType = opaque { TOTALORDER, }; - pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { - return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const CompareTypeAttr { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const CompareType) Type { - const value = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const CompareTypeAttr) Type { + const name = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const CompareTypeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { - return @ptrCast(CompareType.init(ctx, value)); +pub fn compareTypeAttr(ctx: *mlir.Context, value: CompareTypeAttr.Type) *const mlir.Attribute { + return CompareTypeAttr.get(ctx, .{ .value = value }).attribute(); } -pub const TransposeAttribute = opaque { - const M = mlir.Methods(TransposeAttribute, mlir.Attribute); +pub const TransposeAttr = opaque { + const M = mlir.Methods(TransposeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsATransposeAttr; pub const ptr = M.ptr; @@ -1263,22 +1473,30 @@ pub const TransposeAttribute = opaque { ADJOINT, }; - pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { - return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TransposeAttr { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const TransposeAttribute) Type { - const value = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const TransposeAttr) Type { + const name = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const TransposeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { - return @ptrCast(TransposeAttribute.init(ctx, value)); +pub fn transposeAttr(ctx: *mlir.Context, value: TransposeAttr.Type) *const mlir.Attribute { + return TransposeAttr.get(ctx, .{ .value = value }).attribute(); } -pub const FftType = opaque { - const M = mlir.Methods(FftType, mlir.Attribute); +pub const FftTypeAttr = opaque { + const M = mlir.Methods(FftTypeAttr, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; pub const ptr = M.ptr; @@ -1292,18 +1510,26 @@ pub const FftType = opaque { IRFFT, }; - pub fn init(ctx: *mlir.Context, value: Type) *const FftType { - return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const FftTypeAttr { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); } - pub fn getValue(self: *const FftType) Type { - const value = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, value) orelse unreachable; + pub fn value(self: *const FftTypeAttr) Type { + const name = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const FftTypeAttr) *const mlir.Attribute { + return @ptrCast(self); } }; -pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { - return @ptrCast(FftType.init(ctx, value)); +pub fn fftTypeAttr(ctx: *mlir.Context, value: FftTypeAttr.Type) *const mlir.Attribute { + return FftTypeAttr.get(ctx, .{ .value = value }).attribute(); } fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { @@ -1365,7 +1591,9 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const if (sctx.err) |err| { return err; } - if (c.mlirLogicalResultIsFailure(result)) { + // TODO(Corentin): Find out why this crashes on macOS + //if (c.mlirLogicalResultIsFailure(result)) { + if (@as(c_int, result.value) == 0) { return error.InvalidMlirBytecodeVersion; } } @@ -1385,3 +1613,43 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } + +pub const ChannelType = enum(i64) { + collective = 0, + send_recv = 1, +}; + +pub fn channelHandleAttr( + ctx: *mlir.Context, + handle: i64, + channel_type: ChannelType, +) *const mlir.Attribute { + return @ptrCast(c.stablehloChannelHandleGet( + ctx.ptr(), + handle, + @backingInt(channel_type), + ).ptr); +} + +pub fn allReduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + reducer_block: *mlir.Block, + replica_groups: *const mlir.Attribute, + channel_handle: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.all_reduce", .{ + .operands = .{ .variadic = &.{inputs} }, + .results = .{ .flat = result_types }, + .blocks = &.{reducer_block}, + .attributes = &.{ + .named(ctx, "replica_groups", replica_groups), + .named(ctx, "channel_handle", channel_handle), + .named(ctx, "use_global_device_ids", .unit(ctx)), + }, + .verify = true, + .location = location, + }); +} diff --git a/mlir/dialects/stablehlo/test.zig b/mlir/dialects/stablehlo/test.zig new file mode 100644 index 00000000..c22b2ebd --- /dev/null +++ b/mlir/dialects/stablehlo/test.zig @@ -0,0 +1,106 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); + +const operation_names: []const []const u8 = &.{ + "stablehlo.abs", + "stablehlo.add", + "stablehlo.all_reduce", + "stablehlo.and", + "stablehlo.atan2", + "stablehlo.bitcast_convert", + "stablehlo.broadcast_in_dim", + "stablehlo.cbrt", + "stablehlo.ceil", + "stablehlo.cholesky", + "stablehlo.clamp", + "stablehlo.compare", + "stablehlo.complex", + "stablehlo.composite", + "stablehlo.concatenate", + "stablehlo.constant", + "stablehlo.convert", + "stablehlo.convolution", + "stablehlo.cosine", + "stablehlo.count_leading_zeros", + "stablehlo.custom_call", + "stablehlo.divide", + "stablehlo.dot_general", + "stablehlo.dynamic_slice", + "stablehlo.dynamic_update_slice", + "stablehlo.exponential", + "stablehlo.exponential_minus_one", + "stablehlo.fft", + "stablehlo.floor", + "stablehlo.gather", + "stablehlo.get_tuple_element", + "stablehlo.if", + "stablehlo.imag", + "stablehlo.iota", + "stablehlo.is_finite", + "stablehlo.log", + "stablehlo.log_plus_one", + "stablehlo.logistic", + "stablehlo.maximum", + "stablehlo.minimum", + "stablehlo.multiply", + "stablehlo.negate", + "stablehlo.not", + "stablehlo.optimization_barrier", + "stablehlo.or", + "stablehlo.pad", + "stablehlo.partition_id", + "stablehlo.popcnt", + "stablehlo.power", + "stablehlo.real", + "stablehlo.reduce", + "stablehlo.reduce_precision", + "stablehlo.reduce_window", + "stablehlo.remainder", + "stablehlo.reshape", + "stablehlo.return", + "stablehlo.reverse", + "stablehlo.rng", + "stablehlo.rng_bit_generator", + "stablehlo.round_nearest_afz", + "stablehlo.round_nearest_even", + "stablehlo.rsqrt", + "stablehlo.scatter", + "stablehlo.select", + "stablehlo.shift_left", + "stablehlo.shift_right_arithmetic", + "stablehlo.shift_right_logical", + "stablehlo.sign", + "stablehlo.sine", + "stablehlo.slice", + "stablehlo.sort", + "stablehlo.sqrt", + "stablehlo.subtract", + "stablehlo.tanh", + "stablehlo.transpose", + "stablehlo.triangular_solve", + "stablehlo.tuple", + "stablehlo.while", + "stablehlo.xor", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(stablehlo); + inline for (comptime std.meta.declarations(stablehlo)) |decl| { + const T = @field(stablehlo, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + stablehlo.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/tcl/BUILD.bazel b/mlir/dialects/tcl/BUILD.bazel new file mode 100644 index 00000000..d57b6f36 --- /dev/null +++ b/mlir/dialects/tcl/BUILD.bazel @@ -0,0 +1,172 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "tcl", + import_name = "mlir/dialects/tcl", + main = "tcl.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":tcl", + "//mlir", + "//mlir/dialects/func", + ], +) + +# The native subset of the Furiosa XLA plugin's `tcl` dialect +# (xla/backends/furiosa/codegen/tcl/ir): what ZML emits and the plugin's custom +# call parses. Files mirror the plugin's one for one. +td_library( + name = "td_files", + srcs = [ + "tcl_attrs.td", + "tcl_dialect.td", + "tcl_ops.td", + "tcl_types.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + ["-gen-dialect-decls"], + "tcl_dialect.h.inc", + ), + ( + ["-gen-dialect-defs"], + "tcl_dialect.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=tcl", + ], + "tcl_types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=tcl", + ], + "tcl_types.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_types.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_attrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "tcl_ops.h.inc", + ), + ( + ["-gen-op-defs"], + "tcl_ops.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_ops.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "tcl_attrs.h", + "tcl_dialect.h", + "tcl_ops.h", + "tcl_types.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "tcl_attrs.cc", + "tcl_dialect.cc", + "tcl_ops.cc", + "tcl_types.cc", + ], + textual_hdrs = ["ve_instructions.inc"], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/tcl/capi.cc b/mlir/dialects/tcl/capi.cc new file mode 100644 index 00000000..70ee09d6 --- /dev/null +++ b/mlir/dialects/tcl/capi.cc @@ -0,0 +1,118 @@ +//===- capi.cc - C API for the Furiosa TCL dialect ------------------------===// + +#include "capi.h" + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +#include + +using namespace mlir; +using namespace xla::furiosa::tcl; + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Tcl, tcl, TclDialect) + +// `getChecked` with its diagnostic silenced: an invalid parameter is a null +// handle, which the caller reports. +template +static auto checked(MlirContext context, Args &&...args) { + MLIRContext *ctx = unwrap(context); + if (!ctx) + return T(); + ctx->getOrLoadDialect(); + ScopedDiagnosticHandler silence(ctx, [](Diagnostic &) { return success(); }); + return T::getChecked([ctx] { return emitError(UnknownLoc::get(ctx)); }, ctx, + std::forward(args)...); +} + +template static T as(MlirAttribute attr) { + return dyn_cast_if_present(unwrap(attr)); +} + +MlirAttribute mlirTclSymbolAttrGet(MlirContext context, MlirStringRef name) { + return wrap(checked(context, unwrap(name))); +} + +MlirAttribute mlirTclTacticAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclContextAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclReadOptionsAttrGet(MlirContext context, + MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclConfigAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclExprAttrGet(MlirContext context, MlirStringRef kind, + MlirAttribute args) { + auto array = as(args); + return array ? wrap(checked(context, unwrap(kind), array)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, + MlirAttribute original) { + auto c = as(chip), i = as(inner), + o = as(original); + return c && i && o ? wrap(checked(context, c, i, o)) + : MlirAttribute{nullptr}; +} + +MlirType mlirTclLogicalTypeGet(MlirContext context, MlirType element, + MlirAttribute axes) { + auto array = as(axes); + if (!array || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), array)); +} + +bool mlirTypeIsATclLogical(MlirType type) { + return isa_and_nonnull(unwrap(type)); +} + +MlirType mlirTclLogicalTypeGetElementType(MlirType type) { + return wrap(cast(unwrap(type)).getElementType()); +} + +MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type) { + return wrap(cast(unwrap(type)).getAxes()); +} + +MlirType mlirTclMappedTypeGet(MlirContext context, MlirType element, + MlirAttribute mapping) { + auto m = as(mapping); + if (!m || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), m)); +} diff --git a/mlir/dialects/tcl/capi.h b/mlir/dialects/tcl/capi.h new file mode 100644 index 00000000..31e72c3b --- /dev/null +++ b/mlir/dialects/tcl/capi.h @@ -0,0 +1,53 @@ +//===- capi.h - C API for the Furiosa TCL dialect -----------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TCL_CAPI_H +#define MLIR_DIALECTS_TCL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Tcl, tcl); + +// Constructors verify their parameters and return a null handle when invalid. +MLIR_CAPI_EXPORTED MlirAttribute mlirTclSymbolAttrGet(MlirContext context, + MlirStringRef name); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclExprAttrGet(MlirContext context, + MlirStringRef kind, + MlirAttribute args); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclTacticAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclContextAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclReadOptionsAttrGet(MlirContext context, MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclConfigAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, MlirAttribute original); + +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGet(MlirContext context, + MlirType element, + MlirAttribute axes); +MLIR_CAPI_EXPORTED bool mlirTypeIsATclLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGetElementType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type); + +MLIR_CAPI_EXPORTED MlirType mlirTclMappedTypeGet(MlirContext context, + MlirType element, + MlirAttribute mapping); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TCL_CAPI_H diff --git a/mlir/dialects/tcl/tcl.zig b/mlir/dialects/tcl/tcl.zig new file mode 100644 index 00000000..a3eb4673 --- /dev/null +++ b/mlir/dialects/tcl/tcl.zig @@ -0,0 +1,364 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tcl"); +} + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn attribute(a: c.MlirAttribute) mlir.Error!*const mlir.Attribute { + return if (a.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +fn type_(t: c.MlirType) mlir.Error!*const mlir.Type { + return if (t.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +// ============================================================================= +// Attributes +// ============================================================================= + +pub const Tactic = enum { + einsum_by_dpe, + reduce_by_ve, + interleaving, + elementwise, + tensor_operation, + einsum_by_ve, + filter_compaction, + + pub fn name(self: Tactic) []const u8 { + return switch (self) { + .einsum_by_dpe => "EinsumByDpe", + .reduce_by_ve => "ReduceByVe", + .interleaving => "Interleaving", + .elementwise => "Elementwise", + .tensor_operation => "TensorOperation", + .einsum_by_ve => "EinsumByVe", + .filter_compaction => "FilterCompaction", + }; + } +}; + +pub const ReduceMode = enum { + addi, + addf, + maxi, + maxf, + mini, + minf, + /// A running i32 sum that keeps its axes. + cumsum, + + pub fn name(self: ReduceMode) []const u8 { + return switch (self) { + .addi => "Addi", + .addf => "Addf", + .maxi => "Maxi", + .maxf => "Maxf", + .mini => "Mini", + .minf => "Minf", + .cumsum => "Cumsum", + }; + } +}; + +pub const Predicate = enum { eq, ne, lt, le, gt, ge }; + +/// An axis or another TCL identifier: `#tcl.symbol<"A">`. +pub fn symbolAttr(ctx: *mlir.Context, name: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclSymbolAttrGet(ctx.ptr(), stringRef(name))); +} + +/// `#tcl.expr`: `add`, `mul`, `stride`, `broadcast`, `arg`, ... +pub fn exprAttr(ctx: *mlir.Context, kind: []const u8, args: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclExprAttrGet(ctx.ptr(), stringRef(kind), mlir.Attribute.array(ctx, args).ptr())); +} + +pub fn tacticAttr(ctx: *mlir.Context, value: Tactic) *const mlir.Attribute { + return attribute(c.mlirTclTacticAttrGet(ctx.ptr(), stringRef(value.name()))) catch unreachable; +} + +/// A VE opcode of the SDK vocabulary (`exp`, `addf`, `to_f16`, ...). +pub fn veOpcodeAttr(ctx: *mlir.Context, opcode: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclVeOpcodeAttrGet(ctx.ptr(), stringRef(opcode))); +} + +pub fn reduceModeAttr(ctx: *mlir.Context, mode: ReduceMode) *const mlir.Attribute { + return attribute(c.mlirTclReduceModeAttrGet(ctx.ptr(), stringRef(mode.name()))) catch unreachable; +} + +pub fn predicateAttr(ctx: *mlir.Context, value: Predicate) *const mlir.Attribute { + return attribute(c.mlirTclPredicateAttrGet(ctx.ptr(), stringRef(@tagName(value)))) catch unreachable; +} + +/// `@ context(operator = {Chip: ...}, heuristic_hint = {...})`. +pub fn contextAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclContextAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `read(x, subtraction=, table_lookup=, broadcast_to=, typecast_to=, pad=)`. +pub fn readOptionsAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclReadOptionsAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `@compiler_config({...})` / `@auto_config({...})`, JSON-compatible fields. +pub fn configAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclConfigAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `dram(chip | inner)`, with the original axes of a padded mapping. +pub fn dramMappingAttr(ctx: *mlir.Context, chip: []const *const mlir.Attribute, inner: []const *const mlir.Attribute, original: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclDramMappingAttrGet( + ctx.ptr(), + mlir.Attribute.array(ctx, chip).ptr(), + mlir.Attribute.array(ctx, inner).ptr(), + mlir.Attribute.array(ctx, original).ptr(), + )); +} + +/// Discardable TCL attributes of a `func.func`. +pub const FunctionOpts = struct { + /// `tcl.updated_inputs`: per result, the argument it updates in place, or -1. + updated_inputs: ?[]const i64 = null, + /// `tcl.compiler_config`, a `configAttr`. + compiler_config: ?*const mlir.Attribute = null, + /// `tcl.auto_config`, a `configAttr`. + auto_config: ?*const mlir.Attribute = null, + /// `tcl.fusible`: the plugin may compile the function into the unit around it. + fusible: bool = false, +}; + +/// Sets `tcl.axes`, each axis's size or definition by name, and the +/// attributes of `opts` on `function`. +pub fn annotateFunction(ctx: *mlir.Context, function: *mlir.Operation, axes: []const mlir.NamedAttribute, opts: FunctionOpts) void { + if (opts.updated_inputs) |inputs| function.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, inputs)); + function.setAttributeByName("tcl.axes", .dict(ctx, axes)); + if (opts.compiler_config) |config| function.setAttributeByName("tcl.compiler_config", config); + if (opts.auto_config) |config| function.setAttributeByName("tcl.auto_config", config); + if (opts.fusible) function.setAttributeByName("tcl.fusible", .unit(ctx)); +} + +// ============================================================================= +// Types +// ============================================================================= + +/// `[A, B]/bf16`: a logical tensor over named axes. +pub fn logicalType(ctx: *mlir.Context, element: *const mlir.Type, axes: []const *const mlir.Attribute) mlir.Error!*const mlir.Type { + return (try LogicalType.get(ctx, .{ .elementType = element, .axes = axes })).type_(); +} + +/// `dram(...)/bf16`: a tensor in an explicit DRAM mapping. +pub fn mappedType(ctx: *mlir.Context, element: *const mlir.Type, mapping: *const mlir.Attribute) mlir.Error!*const mlir.Type { + return type_(c.mlirTclMappedTypeGet(ctx.ptr(), element.ptr(), mapping.ptr())); +} + +pub const LogicalType = opaque { + const M = mlir.Methods(LogicalType, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const isAFn = c.mlirTypeIsATclLogical; + + pub const InitArgs = struct { + elementType: *const mlir.Type, + axes: []const *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LogicalType { + return @ptrCast(c.mlirTclLogicalTypeGet(ctx.ptr(), args.elementType.ptr(), mlir.Attribute.array(ctx, args.axes).ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn type_(self: *const LogicalType) *const mlir.Type { + return @ptrCast(self); + } + + pub fn elementType(self: *const LogicalType) *const mlir.Type { + return @ptrCast(c.mlirTclLogicalTypeGetElementType(self.ptr()).ptr); + } + + pub fn axes(self: *const LogicalType) *const mlir.ArrayAttribute { + return @ptrCast(c.mlirTclLogicalTypeGetAxes(self.ptr()).ptr); + } +}; + +// ============================================================================= +// Operations +// ============================================================================= +// +// Builders do not verify: kernel instructions only verify inside a +// `tcl.kernel`, and a kernel or graph operation only once its operands +// dominate it. Verify each graph-level operation after inserting it. + +pub const KernelOpts = struct { + context: ?*const mlir.Attribute = null, +}; + +pub fn kernel(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, tactic: Tactic, body: *mlir.Block, opts: KernelOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "tactic", tacticAttr(ctx, tactic)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.kernel", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const ReadOpts = struct { + options: ?*const mlir.Attribute = null, +}; + +pub fn read(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: ReadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.options) |o| attrs[0] = .named(ctx, "options", o); + return mlir.Operation.make(ctx, "tcl.read", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0..@intFromBool(opts.options != null)], + .verify = false, + .location = location, + }); +} + +pub fn dpe(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, output: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.dpe", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{output} }, + .verify = false, + .location = location, + }); +} + +pub fn ve(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opcode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = &.{.named(ctx, "opcode", opcode)}, + .verify = false, + .location = location, + }); +} + +pub fn ve_reduce(ctx: *mlir.Context, input: *const mlir.Value, output: *const mlir.Type, mode: ReduceMode, axes: []const *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "mode", reduceModeAttr(ctx, mode)), + .named(ctx, "axes", .array(ctx, axes)), + }, + .verify = false, + .location = location, + }); +} + +pub fn ve_select(ctx: *mlir.Context, condition: *const mlir.Value, true_value: *const mlir.Value, false_value: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_select", .{ + .operands = .{ .flat = &.{ condition, true_value, false_value } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + }, + .verify = false, + .location = location, + }); +} + +/// `tcl.filter_compaction`: keeps the elements of `input` along `axes` whose +/// `condition` satisfies `predicate` against `threshold`, packed into at most +/// `max_size` (an integer or axis attribute) elements. +pub fn filter_compaction(ctx: *mlir.Context, input: *const mlir.Value, condition: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, axes: []const *const mlir.Attribute, max_size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.filter_compaction", .{ + .operands = .{ .flat = &.{ input, condition } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + .named(ctx, "axes", .array(ctx, axes)), + .named(ctx, "max_size", max_size), + }, + .verify = false, + .location = location, + }); +} + +pub fn write(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.write", .{ + .operands = .{ .flat = &.{input} }, + .verify = false, + .location = location, + }); +} + +pub const GraphOp = enum { gather, all_gather, scatter, reshape, transmute, concat, slice, arange, vector, as_logical, as_dram, index_read, index_write, scratchpad, full, reduce_max_i32, sym_expr }; + +pub const GraphOpts = struct { + options: []const mlir.NamedAttribute = &.{}, + context: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.`: a graph-level TCL operation whose fields go in `options`. +pub fn graph(ctx: *mlir.Context, comptime op: GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: GraphOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "options", .dict(ctx, opts.options)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.graph." ++ @tagName(op), .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .verify = false, + .location = location, + }); +} + +/// `tcl.graph.copy`, `copy(src=value, dst=input)`: writes `src` into the +/// kernel input `dst` in place. +pub fn copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.copy", .{ + .operands = .{ .flat = &.{ src, dst } }, + .results = .{ .flat = &.{} }, + .attributes = &.{.named(ctx, "options", .dict(ctx, &.{}))}, + .verify = false, + .location = location, + }); +} + +pub const LoopOpts = struct { + limit: ?*const mlir.Value = null, + bound: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.for`: runs `body` up to `limit` (a scalar i32 tensor) or `bound` +/// (an integer or axis attribute) times. The body's arguments are the index, +/// then one accumulator per `inits`; it ends with `tcl.graph.yield`. +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, opts: LoopOpts, location: *const mlir.Location) *mlir.Operation { + const limits: []const *const mlir.Value = if (opts.limit) |l| &.{l} else &.{}; + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.bound) |b| attrs[0] = .named(ctx, "bound", b); + return mlir.Operation.make(ctx, "tcl.graph.for", .{ + .operands = .{ .variadic = &.{ limits, inits } }, + .results = .{ .flat = result_types }, + .attributes = attrs[0..@intFromBool(opts.bound != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} diff --git a/mlir/dialects/tcl/tcl_attrs.cc b/mlir/dialects/tcl/tcl_attrs.cc new file mode 100644 index 00000000..5b2fe575 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.cc @@ -0,0 +1,266 @@ +//===- tcl_attrs.cc - Furiosa TCL attributes ------------------------------===// + +#include "mlir/dialects/tcl/tcl_attrs.h" + +#include "mlir/dialects/tcl/tcl_dialect.h" +#include "mlir/dialects/tcl/tcl_types.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" + +#include +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +static bool Identifier(llvm::StringRef s) { + if (s.empty() || (!llvm::isAlpha(s.front()) && s.front() != '_')) + return false; + return llvm::all_of(s, [](char c) { return llvm::isAlnum(c) || c == '_'; }); +} +static bool Symbol(llvm::StringRef s) { + auto [base, view] = s.split('.'); + return Identifier(base) && + (view.empty() ? !s.contains('.') : Identifier(view)); +} +static bool Mapping(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + auto e = dyn_cast(a); + return e && + llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair", "broadcast"}, + e.getKind()) && + llvm::all_of(e.getArgs(), Mapping); +} +static bool Json(Attribute a) { + if (auto n = dyn_cast(a)) + return n.getValue().isFinite(); + if (IsNumber(a) || isa(a)) + return true; + if (auto arr = dyn_cast(a)) + return llvm::all_of(arr, Json); + if (auto dict = dyn_cast(a)) + return llvm::all_of(dict, + [](NamedAttribute f) { return Json(f.getValue()); }); + return false; +} +static LogicalResult Enum(Error error, llvm::StringRef value, + llvm::ArrayRef values) { + return llvm::is_contained(values, value) + ? success() + : error() << "invalid enumerator: " << value; +} + +bool IsNumber(Attribute a) { + if (!a) + return false; + if (auto i = dyn_cast(a)) + return i.getValue().getBitWidth() <= 64; + return isa(a); +} +bool IsExpression(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + if (auto e = dyn_cast(a)) + return e.getKind() != "broadcast" && e.getKind() != "pair" && + e.getKind() != "padding" && e.getKind() != "resize"; + return false; +} +bool IsAxis(Attribute a) { + if (!a) + return false; + auto s = dyn_cast(a); + return s && llvm::isUpper(s.getName().front()); +} +bool IsAxes(Attribute a) { + if (!a) + return false; + auto array = dyn_cast(a); + return array && llvm::all_of(array, IsAxis); +} +LogicalResult VerifyKeys(Error error, DictionaryAttr d, + llvm::ArrayRef keys) { + for (auto f : d) + if (!llvm::is_contained(keys, f.getName().strref())) + return error() << "unknown field: " << f.getName(); + return success(); +} +LogicalResult VerifyAxes(Error error, ArrayAttr axes) { + if (!IsAxes(axes)) + return error() << "axes must be uppercase TCL symbols"; + llvm::SmallDenseSet seen; + for (auto a : axes) + if (!seen.insert(a).second) + return error() << "duplicate logical axis"; + return success(); +} + +// opcode, emitted spelling, arity. Conversion widths are handled below. +static constexpr VeInstructionSpec kVeInstructions[] = { +#define VE(name, text, arity) {name, text, arity}, +#include "mlir/dialects/tcl/ve_instructions.inc" +#undef VE +}; +llvm::ArrayRef VeInstructions() { return kVeInstructions; } + +LogicalResult SymbolAttr::verify(Error error, llvm::StringRef name) { + return Symbol(name) ? success() + : error() << "expected a TCL identifier, optionally with " + "one view suffix"; +} +LogicalResult ExprAttr::verify(Error error, llvm::StringRef kind, + ArrayAttr args) { + if (kind == "broadcast") + return args.empty() ? success() : error() << "broadcast takes no arguments"; + if (kind == "arg") { + if (args.size() != 1 || !isa(args[0]) || !IsNumber(args[0]) || + cast(args[0]).getInt() < 0) + return error() << "arg requires one nonnegative operand index"; + return success(); + } + bool map = llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair"}, kind); + bool arithmetic = + llvm::is_contained({"add", "sub", "mul", "div", "rem", "exact_div", "eq", + "ne", "lt", "le", "gt", "ge"}, + kind); + if ((!map && !arithmetic) || args.size() != 2) + return error() << "expected a known binary expression"; + for (auto a : args) + if (!(map ? Mapping(a) : IsExpression(a))) + return error() << "invalid expression operand"; + if (llvm::is_contained({"div", "exact_div", "rem", "stride", "modulo"}, kind)) + if (auto n = dyn_cast(args[1]); n && n.getInt() == 0) + return error() << "division by zero"; + return success(); +} +LogicalResult TacticAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, + {"EinsumByDpe", "ReduceByVe", "Interleaving", "Elementwise", + "TensorOperation", "EinsumByVe", "FilterCompaction"}); +} +LogicalResult VeOpcodeAttr::verify(Error e, llvm::StringRef v) { + for (const auto &spec : VeInstructions()) + if (spec.name == v) + return success(); + if (v.consume_front("to_f") || v.consume_front("to_i")) { + unsigned width; + if (!v.getAsInteger(10, width) && width <= 31 && std::to_string(width) == v) + return success(); + } + return e() << "unknown VE opcode"; +} +LogicalResult ReduceModeAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"Addi", "Addf", "Maxi", "Maxf", "Mini", "Minf", "Cumsum"}); +} + +LogicalResult PredicateAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"eq", "ne", "lt", "le", "gt", "ge"}); +} +LogicalResult ConfigAttr::verify(Error e, DictionaryAttr d) { + return Json(d) ? success() + : e() << "compiler/auto configuration requires " + "JSON-compatible attributes"; +} + +LogicalResult ReadOptionsAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, + {"subtraction", "table_lookup", "broadcast_to", + "typecast_to", "pad", "slide"}))) + return failure(); + for (auto field : d) { + auto k = field.getName().strref(); + auto v = field.getValue(); + if (k == "subtraction" || k == "table_lookup") { + auto i = dyn_cast(v); + if (!i || !IsNumber(i) || i.getInt() < 1) + return e() << k << " must index an auxiliary read operand (>=1)"; + } else if (k == "broadcast_to") { + if (!IsAxes(v)) + return e() << "broadcast_to requires axes"; + } else if (k == "typecast_to") { + auto t = dyn_cast(v); + if (!t || ElementName(t.getValue()).empty()) + return e() << "invalid typecast element type"; + } else if (k == "slide") { + auto entries = dyn_cast(v); + if (!entries) + return e() << "slide requires an axis dictionary"; + for (auto item : entries) { + auto spec = dyn_cast(item.getValue()); + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front()) || !spec || + failed(VerifyKeys(e, spec, + {"undilated_window", "frame_axis", "window_axis", + "stride", "dilation"})) || + !IsExpression(spec.get("undilated_window")) || + !IsAxis(spec.get("frame_axis")) || !IsAxis(spec.get("window_axis"))) + return e() << "slide requires undilated_window, frame_axis and " + "window_axis"; + for (auto key : {"stride", "dilation"}) + if (auto a = spec.get(key); a && !IsExpression(a)) + return e() << "invalid slide " << key; + } + } else { + auto entries = dyn_cast(v); + if (!entries) + return e() << k << " requires an axis dictionary"; + for (auto item : entries) { + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front())) + return e() << "invalid axis"; + auto p = dyn_cast(item.getValue()); + if (!p || p.size() != 3 || !IsExpression(p[0]) || !IsExpression(p[1]) || + !IsNumber(p[2])) + return e() << "padding requires [left, right, fill]"; + } + } + } + return success(); +} +LogicalResult ContextAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, {"operator", "heuristic_hint"}))) + return failure(); + if (auto a = d.get("operator")) { + auto layout = dyn_cast(a); + if (!layout || failed(VerifyKeys(e, layout, {"Chip", "Cluster", "Split"}))) + return e() << "invalid operator layout"; + if (!layout.get("Chip")) + return e() << "operator layout requires Chip"; + for (auto f : layout) { + if (f.getName() == "Split") { + if (!IsAxes(f.getValue())) + return e() << "Split requires axes"; + } else if (!IsAxis(f.getValue()) && + !(isa(f.getValue()) && + cast(f.getValue()).getKind() == "broadcast")) + return e() << "Chip/Cluster requires axis or Broadcast"; + } + } + if (auto a = d.get("heuristic_hint")) { + auto hints = dyn_cast(a); + if (!hints) + return e() << "heuristic_hint requires a dictionary"; + for (auto f : hints) + if (!Identifier(f.getName()) || + !(IsNumber(f.getValue()) || IsAxis(f.getValue()))) + return e() << "invalid heuristic hint"; + } + return success(); +} + +LogicalResult DramMappingAttr::verify(Error e, ArrayAttr chip, ArrayAttr inner, + ArrayAttr original) { + if (!llvm::all_of(chip, Mapping) || !llvm::all_of(inner, Mapping)) + return e() << "invalid DRAM mapping"; + return VerifyAxes(e, original); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_attrs.h b/mlir/dialects/tcl/tcl_attrs.h new file mode 100644 index 00000000..816583b9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.h @@ -0,0 +1,37 @@ +//===- tcl_attrs.h - Furiosa TCL attributes ---------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_H +#define MLIR_DIALECTS_TCL_TCL_ATTRS_H + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/Attributes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.h.inc" +namespace xla::furiosa::tcl { +struct VeInstructionSpec { + llvm::StringRef name; + llvm::StringRef spelling; + unsigned arity; +}; +llvm::ArrayRef VeInstructions(); + +// Schema predicates shared by attribute, type and operation verifiers. +bool IsNumber(mlir::Attribute a); +bool IsExpression(mlir::Attribute a); +bool IsAxis(mlir::Attribute a); +bool IsAxes(mlir::Attribute a); +mlir::LogicalResult +VerifyKeys(llvm::function_ref error, + mlir::DictionaryAttr d, llvm::ArrayRef keys); +mlir::LogicalResult +VerifyAxes(llvm::function_ref error, + mlir::ArrayAttr axes); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_H diff --git a/mlir/dialects/tcl/tcl_attrs.td b/mlir/dialects/tcl/tcl_attrs.td new file mode 100644 index 00000000..f85c518e --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.td @@ -0,0 +1,74 @@ +//===- tcl_attrs.td - Furiosa TCL attributes ---------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_TD +#define MLIR_DIALECTS_TCL_TCL_ATTRS_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Symbol : AttrDef { + let mnemonic = "symbol"; + let parameters = (ins StringRefParameter<>:$name); + let assemblyFormat = "`<` $name `>`"; + let genVerifyDecl = 1; +} +def Tcl_Expr : AttrDef { + let mnemonic = "expr"; + let summary = "Checked symbolic arithmetic or DRAM mapping expression tree"; + let parameters = (ins StringRefParameter<>:$kind, "::mlir::ArrayAttr":$args); + let assemblyFormat = "`<` $kind `,` $args `>`"; + let genVerifyDecl = 1; +} +def Tcl_Tactic : AttrDef { + let mnemonic = "tactic"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_VeOpcode : AttrDef { + let mnemonic = "ve_opcode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReduceMode : AttrDef { + let mnemonic = "reduce_mode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Predicate : AttrDef { + let mnemonic = "predicate"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_Context : AttrDef { + let mnemonic = "context"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReadOptions : AttrDef { + let mnemonic = "read_options"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Config : AttrDef { + let mnemonic = "config"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_DramMapping : AttrDef { + let mnemonic = "dram_mapping"; + let parameters = (ins "::mlir::ArrayAttr":$chip, "::mlir::ArrayAttr":$inner, + "::mlir::ArrayAttr":$original); + let assemblyFormat = "`<` $chip `,` $inner `,` $original `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_TD diff --git a/mlir/dialects/tcl/tcl_dialect.cc b/mlir/dialects/tcl/tcl_dialect.cc new file mode 100644 index 00000000..2cea51a1 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.cc @@ -0,0 +1,34 @@ +//===- tcl_dialect.cc - Furiosa TCL dialect -------------------------------===// + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_ops.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: keep + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/DialectImplementation.h" // IWYU pragma: keep +#include "llvm/ADT/TypeSwitch.h" // IWYU pragma: keep + +// Storage definitions must be complete where the dialect registers them. +#include "mlir/dialects/tcl/tcl_dialect.cc.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.cc.inc" +namespace xla::furiosa::tcl { +void TclDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/tcl/tcl_types.cc.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/tcl/tcl_ops.cc.inc" + >(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_dialect.h b/mlir/dialects/tcl/tcl_dialect.h new file mode 100644 index 00000000..97267065 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.h @@ -0,0 +1,9 @@ +//===- tcl_dialect.h - Furiosa TCL dialect ----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_H +#define MLIR_DIALECTS_TCL_TCL_DIALECT_H + +#include "mlir/IR/Dialect.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_dialect.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_H diff --git a/mlir/dialects/tcl/tcl_dialect.td b/mlir/dialects/tcl/tcl_dialect.td new file mode 100644 index 00000000..7b86a825 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.td @@ -0,0 +1,19 @@ +//===- tcl_dialect.td - Furiosa TCL dialect ----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_TD +#define MLIR_DIALECTS_TCL_TCL_DIALECT_TD + +include "mlir/IR/DialectBase.td" + +// Native TCL: tensors over named axes, `tk.*` tensor operations and graph +// operations, one-to-one with the TCL source the Furiosa compiler takes. The +// subset of xla/backends/furiosa/codegen/tcl/ir that ZML emits. +def Tcl_Dialect : Dialect { + let name = "tcl"; + let cppNamespace = "::xla::furiosa::tcl"; + let summary = "Typed Furiosa Tensor Compute Language graph"; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_TD diff --git a/mlir/dialects/tcl/tcl_ops.cc b/mlir/dialects/tcl/tcl_ops.cc new file mode 100644 index 00000000..9d569f0b --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.cc @@ -0,0 +1,473 @@ +//===- tcl_ops.cc - Furiosa TCL operations --------------------------------===// + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.cc.inc" +namespace xla::furiosa::tcl { +using namespace mlir; +static bool Tensor(Type t) { return isa(t); } +static bool ScalarI32(Type t) { + auto l = dyn_cast(t); + return l && l.getAxes().empty() && l.getElementType().isInteger(32); +} +static bool LoopIndex(Value v) { + auto arg = dyn_cast(v); + return arg && arg.getArgNumber() == 0 && + isa_and_nonnull(arg.getOwner()->getParentOp()); +} +static LogicalResult InKernel(Operation *op) { + if (!op->getParentOfType()) + return op->emitOpError("requires a tcl.kernel region"); + return success(); +} +// Exact graph operation schema. Optional operands occupy documented positions; +// the options dictionary never changes operand ordering. +static LogicalResult VerifyGraph(Operation *op, DictionaryAttr options) { + auto e = [&] { return op->emitOpError(); }; + llvm::StringRef n = op->getName().getStringRef().drop_front(10); + int min = 1, max = 1, results = 1; + llvm::SmallVector fields; + if (n == "all_gather") { + fields = {"axis"}; + } else if (n == "gather" || n == "scatter") { + min = 2; + max = 3; + fields = {"axis", "batch_axis", "table_dram_layout"}; + } else if (n == "concat") { + max = -1; + } else if (n == "slice") { + fields = {"axis", "offset"}; + } else if (n == "arange") { + min = max = 0; + fields = {"start", "end", "step"}; + } else if (n == "vector") { + min = max = 0; + fields = {"values", "repeat_to"}; + } else if (n == "index_read") { + min = max = 2; + } else if (n == "index_write") { + min = max = 3; + } else if (n == "copy") { + min = max = 2; + results = 0; + } else if (n == "scratchpad") { + min = max = 0; + } else if (n == "full") { + min = max = 0; + fields = {"value"}; + } else if (n == "sym_expr") { + min = 0; + max = -1; + fields = {"expr"}; + } + if (op->getNumOperands() < min || (max >= 0 && op->getNumOperands() > max) || + op->getNumResults() != results) + return e() << "incorrect operand or result count"; + if (failed(VerifyKeys(e, options, fields))) + return failure(); + if (n == "gather" && options.get("batch_axis") && op->getNumOperands() == 3) + return e() << "SDK sparse gather cannot also use batch_axis"; + for (Type t : op->getOperandTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor operand"; + for (Type t : op->getResultTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor result"; + for (auto f : options) { + auto k = f.getName().strref(); + auto v = f.getValue(); + bool ok = false; + if (k.ends_with("axis") || k == "repeat_to") + ok = IsAxis(v); + else if (k == "axes") + ok = IsAxes(v); + else if (k == "value") + ok = IsNumber(v); + else if (k == "values") { + auto a = dyn_cast(v); + ok = a && !a.empty() && llvm::all_of(a, IsNumber); + } else if (k == "table_dram_layout") + ok = isa(v) && isa(cast(v).getValue()); + else + ok = IsExpression(v); + if (!ok) + return e() << "invalid field " << k; + } + for (auto k : fields) { + bool required = llvm::is_contained( + {"axis", "offset", "end", "values", "expr", "value"}, k); + if (required && !options.get(k)) + return e() << "missing required field " << k; + } + if (n == "as_logical" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_logical requires storage -> logical"; + if (n == "as_dram" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_dram requires logical -> mapped"; + if (n == "index_read" || n == "index_write") { + if (!LoopIndex(op->getOperand(1))) + return e() << "requires the index of an enclosing loop"; + auto table = dyn_cast(op->getOperand(0).getType()); + auto slice = dyn_cast( + (n == "index_read" ? op->getResult(0) : op->getOperand(2)).getType()); + if (!table || !slice || table.getAxes().empty() || + table.getElementType() != slice.getElementType() || + table.getAxes().getValue().drop_front() != slice.getAxes().getValue()) + return e() << "slice must be the table without its outermost axis"; + if (n == "index_write" && + (op->getResult(0).getType() != table || !op->getOperand(0).hasOneUse())) + return e() << "writes in place: the table must have no other use"; + } + if (n == "copy" && + (!isa(op->getOperand(1)) || + op->getOperand(0).getType() != op->getOperand(1).getType())) + return e() << "copies a value into a kernel input of its type"; + if ((n == "reduce_max_i32" || n == "sym_expr") && + !ScalarI32(op->getResult(0).getType())) + return e() << "requires a scalar i32 result"; + if (n == "sym_expr") { + for (Value v : op->getOperands()) + if (!ScalarI32(v.getType())) + return e() << "requires scalar i32 operands"; + llvm::SmallVector pending{options.get("expr")}; + while (!pending.empty()) { + auto x = dyn_cast(pending.pop_back_val()); + if (!x) + continue; + if (x.getKind() == "arg") { + if (cast(x.getArgs()[0]).getInt() >= op->getNumOperands()) + return e() << "expression operand out of range"; + continue; + } + pending.append(x.getArgs().begin(), x.getArgs().end()); + } + } + return success(); +} + +LogicalResult KernelOp::verify() { + if (getBody().empty() || getBody().front().empty() || + getBody().front().getNumArguments()) + return emitOpError("requires one body without block arguments"); + auto write = dyn_cast(getBody().front().getTerminator()); + if (!write) + return emitOpError("requires tcl.write terminator"); + for (auto input : getInputs()) + if (!Tensor(input.getType())) + return emitOpError("requires native tensor inputs"); + for (Operation &op : getBody().front()) { + if (!isa(op) && + op.getName().getStringRef() != "arith.constant") + return emitOpError("unsupported kernel instruction ") << op.getName(); + } + WalkResult captured = getBody().walk([&](Operation *op) { + for (Value v : op->getOperands()) { + if (getBody().isAncestor(v.getParentRegion())) + continue; + if (!isa(op) || !llvm::is_contained(getInputs(), v)) { + op->emitOpError( + "external values must be explicit kernel inputs consumed by read"); + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + if (captured.wasInterrupted()) + return failure(); + return success(); +} +LogicalResult ReadOp::verify() { + if (failed(InKernel(*this))) + return failure(); + if (getInputs().empty() || + !llvm::all_of(getInputs(), [](Value v) { return Tensor(v.getType()); })) + return emitOpError("requires tensor inputs"); + if (getOptions()) + for (auto k : {"subtraction", "table_lookup"}) + if (auto a = getOptions()->getFields().getAs(k); + a && a.getInt() >= getInputs().size()) + return emitOpError("auxiliary read operand index out of bounds"); + if (auto input = dyn_cast(getInputs()[0].getType())) { + SmallVector axes(input.getAxes().begin(), input.getAxes().end()); + Type element = input.getElementType(); + if (getOptions()) { + auto fields = getOptions()->getFields(); + if (auto table = fields.getAs("table_lookup")) { + auto t = dyn_cast(getInputs()[table.getInt()].getType()); + if (!t || t.getAxes().size() != 1) + return emitOpError("lookup table must be a one-axis logical tensor"); + element = t.getElementType(); + } + if (auto cast = fields.getAs("typecast_to")) + element = cast.getValue(); + if (auto pads = fields.getAs("pad")) + for (auto pad : pads) + if (!llvm::is_contained(axes, + SymbolAttr::get(getContext(), pad.getName()))) + return emitOpError("padding axis missing from input"); + // A slide replaces its axis with the window and frame axes. + if (auto slides = fields.getAs("slide")) + for (auto slide : slides) { + auto it = + llvm::find(axes, SymbolAttr::get(getContext(), slide.getName())); + if (it == axes.end()) + return emitOpError("slide axis missing from input"); + auto spec = cast(slide.getValue()); + *it = spec.get("window_axis"); + axes.insert(it + 1, spec.get("frame_axis")); + } + if (auto broadcast = fields.getAs("broadcast_to")) { + for (auto axis : axes) + if (!llvm::is_contained(broadcast, axis)) + return emitOpError("broadcast must retain all read axes"); + axes.assign(broadcast.begin(), broadcast.end()); + } + } + if (getOutput().getType().getAxes().getValue() != + ArrayRef(axes) || + getOutput().getType().getElementType() != element) + return emitOpError("read result disagrees with its modifiers"); + } + return success(); +} +LogicalResult DpeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + auto out = getOutput().getType(); + if (!out.getElementType().isF32() && + !out.getElementType().isSignlessInteger(32)) + return emitOpError("DPE accumulators require f32 or i32"); + for (auto axis : out.getAxes()) + if (!llvm::is_contained(getLhs().getType().getAxes(), axis) && + !llvm::is_contained(getRhs().getType().getAxes(), axis)) + return emitOpError("DPE result axis missing from both inputs"); + return success(); +} +LogicalResult VeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + unsigned arity = 1; + for (auto spec : VeInstructions()) + if (spec.name == getOpcode().getValue()) + arity = spec.arity; + if (getInputs().size() != arity) + return emitOpError("incorrect VE operand count"); + llvm::SmallDenseSet axes; + for (Value v : getInputs()) { + if (auto type = dyn_cast(v.getType())) { + for (auto a : type.getAxes()) + axes.insert(a); + } else if (!isa(v.getType()) || + !matchPattern(v, m_Constant())) + return emitOpError("VE scalar operands must be constants"); + } + auto out = getOutput().getType(); + if (axes.size() != out.getAxes().size() || + !llvm::all_of(out.getAxes(), + [&](Attribute a) { return axes.contains(a); })) + return emitOpError("VE output axes must be the union of operand axes"); + auto code = getOpcode().getValue(); + llvm::StringRef spelling = code; + for (auto spec : VeInstructions()) + if (spec.name == code) + spelling = spec.spelling; + bool fp = spelling.starts_with("to_f") || spelling == "as_f" || + spelling.ends_with("_f") || + llvm::is_contained({"+", "-", "*", "/", "exp", "neg_exp", "sqrt", + "tanh", "sigmoid", "erf", "log", "sin", "cos"}, + spelling); + if (fp ? !out.getElementType().isF32() + : !out.getElementType().isSignlessInteger(32)) + return emitOpError("VE registers produce f32 or i32 according to opcode"); + return success(); +} +LogicalResult VeReduceOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + // An integer cumulative sum runs along its axes and keeps them. + if (getMode().getValue() == "Cumsum") { + if (getAxes().empty() || + getInput().getType().getAxes() != getOutput().getType().getAxes() || + !llvm::all_of(getAxes(), + [&](Attribute a) { + return llvm::is_contained( + getInput().getType().getAxes(), a); + }) || + !getInput().getType().getElementType().isInteger(32)) + return emitOpError("cumsum keeps the axes of an i32 value"); + return success(); + } + llvm::SmallVector remaining; + for (Attribute a : getInput().getType().getAxes()) + if (!llvm::is_contained(getAxes(), a)) + remaining.push_back(a); + if (getAxes().empty() || + remaining.size() + getAxes().size() != + getInput().getType().getAxes().size() || + remaining != getOutput().getType().getAxes().getValue()) + return emitOpError("reduction result must remove exactly the reduced axes"); + return success(); +} + +LogicalResult VeSelectOp::verify() { + if (failed(InKernel(*this)) || !IsNumber(getThreshold())) + return emitOpError("requires kernel and numeric threshold"); + llvm::SmallDenseSet axes; + for (Value v : + SmallVector{getCondition(), getTrueValue(), getFalseValue()}) { + if (auto t = dyn_cast(v.getType())) { + for (auto a : t.getAxes()) + axes.insert(a); + } else if (!matchPattern(v, m_Constant())) + return emitOpError("scalar branches require constants"); + } + auto output = getOutput().getType().getAxes(); + if (axes.size() != output.size() || + !llvm::all_of(output, [&](Attribute a) { return axes.contains(a); })) + return emitOpError("select output axes must cover its operands"); + return success(); +} + +LogicalResult FilterCompactionOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + auto size = dyn_cast(getMaxSize()); + if (!IsNumber(getThreshold()) || + (!IsAxis(getMaxSize()) && !(size && size.getInt() > 0)) || + getAxes().empty()) + return emitOpError( + "requires numeric threshold, positive maximum size, and axes"); + if (getInput().getType().getAxes() != getCondition().getType().getAxes() || + getInput().getType().getElementType() != + getOutput().getType().getElementType() || + !llvm::all_of(getAxes(), [&](Attribute axis) { + return llvm::is_contained(getInput().getType().getAxes(), axis); + })) + return emitOpError("incompatible compaction input, condition, or axes"); + if (!llvm::all_of(getOutput().getUsers(), + [](Operation *op) { return isa(op); })) + return emitOpError( + "filter_compaction must be the final kernel instruction"); + return success(); +} + +LogicalResult WriteOp::verify() { + auto kernel = dyn_cast((*this)->getParentOp()); + if (!kernel) + return emitOpError("requires a kernel parent"); + auto in = getInput().getType(), out = kernel.getOutput().getType(); + if (in.getAxes().size() != out.getAxes().size()) + return emitOpError("write must preserve the axis set"); + for (auto a : in.getAxes()) + if (!llvm::is_contained(out.getAxes(), a)) + return emitOpError("write must preserve the axis set"); + // TCL write.to implicitly consumes the preceding instruction. + auto previous = (*this)->getPrevNode(); + while (previous && previous->getName().getStringRef() == "arith.constant") + previous = previous->getPrevNode(); + if (getInput().getDefiningOp() != previous) + return emitOpError( + "write must consume the immediately preceding instruction"); + return success(); +} + +LogicalResult GraphGatherOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAllGatherOp::verify() { + if (failed(VerifyGraph(*this, getOptions()))) + return failure(); + auto input = dyn_cast(getInputs()[0].getType()); + auto output = dyn_cast(getOutputs()[0].getType()); + if (!input || !output || input != output) + return emitOpError("all_gather requires matching logical tensor types"); + if (!llvm::is_contained(input.getAxes(), getOptions().get("axis"))) + return emitOpError("collective axis must be present in the input"); + return success(); +} +LogicalResult GraphScatterOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphReshapeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphTransmuteOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphConcatOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphSliceOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphArangeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphVectorOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphAsLogicalOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAsDramOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphCopyOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphIndexReadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphIndexWriteOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphScratchpadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphFullOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphReduceMaxI32Op::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphSymExprOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphForOp::verify() { + if (!getLimit() == !getBound()) + return emitOpError("requires exactly one of a limit tensor or a bound"); + if (getLimit() && !ScalarI32(getLimit().getType())) + return emitOpError("limit must be a scalar i32 tensor"); + if (auto bound = getBoundAttr()) { + auto n = dyn_cast(bound); + if (!IsAxis(bound) && !(n && IsNumber(n) && n.getInt() > 0)) + return emitOpError("bound must be a positive integer or an axis"); + } + Block &body = getBody().front(); + if (body.getNumArguments() != getInits().size() + 1 || + !ScalarI32(body.getArgument(0).getType())) + return emitOpError("requires a scalar i32 index and one accumulator each"); + auto yield = dyn_cast(body.getTerminator()); + if (!yield) + return emitOpError("requires a tcl.graph.yield terminator"); + if (getOutputs().size() != getInits().size() || + yield.getValues().size() != getInits().size()) + return emitOpError("requires one result and one yield per accumulator"); + for (auto [i, init] : llvm::enumerate(getInits())) { + Type t = init.getType(); + if (!Tensor(t) || body.getArgument(i + 1).getType() != t || + getOutputs()[i].getType() != t || yield.getValues()[i].getType() != t) + return emitOpError("accumulator ") << i << " changes type"; + } + return success(); +} + +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_ops.h b/mlir/dialects/tcl/tcl_ops.h new file mode 100644 index 00000000..bce6ccae --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.h @@ -0,0 +1,19 @@ +//===- tcl_ops.h - Furiosa TCL operations -----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_H +#define MLIR_DIALECTS_TCL_TCL_OPS_H + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_dialect.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: export + +#include "mlir/Bytecode/BytecodeOpInterface.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinAttributes.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinTypes.h" // IWYU pragma: keep +#include "mlir/IR/OpDefinition.h" // IWYU pragma: keep +#include "mlir/IR/OpImplementation.h" // IWYU pragma: keep +#include "mlir/Interfaces/SideEffectInterfaces.h" // IWYU pragma: keep +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_H diff --git a/mlir/dialects/tcl/tcl_ops.td b/mlir/dialects/tcl/tcl_ops.td new file mode 100644 index 00000000..23676ec9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.td @@ -0,0 +1,122 @@ +//===- tcl_ops.td - Furiosa TCL operations -----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_TD +#define MLIR_DIALECTS_TCL_TCL_OPS_TD + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/tcl/tcl_attrs.td" +include "mlir/dialects/tcl/tcl_dialect.td" +include "mlir/dialects/tcl/tcl_types.td" + +class Tcl_Op traits = []> + : Op; + +// Options contain typed attributes, never snippets of TCL source. Their exact +// per-operation schema is verified in tcl_ops.cc. Generic assembly is +// intentional: all native constructs support standard MLIR parsing, printing +// and bytecode. +class Tcl_GraphOp traits = [MemoryEffects<[MemRead]>]> + : Tcl_Op { + let arguments = (ins Variadic:$inputs, + DefaultValuedAttr:$options, + OptionalAttr:$context); + let results = (outs Variadic:$outputs); + let hasVerifier = 1; +} +def Tcl_GraphGatherOp : Tcl_GraphOp<"gather">; +def Tcl_GraphAllGatherOp : Tcl_GraphOp<"all_gather">; +def Tcl_GraphScatterOp : Tcl_GraphOp<"scatter">; +def Tcl_GraphReshapeOp : Tcl_GraphOp<"reshape">; +def Tcl_GraphTransmuteOp : Tcl_GraphOp<"transmute">; + +def Tcl_GraphConcatOp : Tcl_GraphOp<"concat">; + +def Tcl_GraphSliceOp : Tcl_GraphOp<"slice">; +def Tcl_GraphArangeOp : Tcl_GraphOp<"arange">; +def Tcl_GraphVectorOp : Tcl_GraphOp<"vector">; + +def Tcl_GraphAsLogicalOp : Tcl_GraphOp<"as_logical">; +def Tcl_GraphAsDramOp : Tcl_GraphOp<"as_dram">; +// `copy(src=value, dst=input)` writes a value into a kernel input in place. +def Tcl_GraphCopyOp + : Tcl_GraphOp<"copy", [MemoryEffects<[MemRead, MemWrite]>]>; + +// Loop-level graph constructs. `x[i]` reads and `acc[i] = v` writes the slice +// of the outermost axis at the loop index; `sym_expr` evaluates an integer +// expression (`#tcl.expr` names its operands) on the device. +def Tcl_GraphIndexReadOp : Tcl_GraphOp<"index_read">; +def Tcl_GraphIndexWriteOp : Tcl_GraphOp<"index_write">; +def Tcl_GraphScratchpadOp : Tcl_GraphOp<"scratchpad">; +def Tcl_GraphFullOp : Tcl_GraphOp<"full">; +def Tcl_GraphReduceMaxI32Op : Tcl_GraphOp<"reduce_max_i32">; +def Tcl_GraphSymExprOp : Tcl_GraphOp<"sym_expr">; + +// `outputs = for i in 0..limit (acc = init, ...): body; yield ...;`. The limit +// is a static bound (integer or axis) or a scalar i32 tensor computed in the +// function. The block arguments are the loop index, then the accumulators. +def Tcl_GraphForOp : Tcl_Op<"graph.for", + [AttrSizedOperandSegments, RecursiveMemoryEffects, SingleBlock]> { + let arguments = (ins Optional:$limit, Variadic:$inits, + OptionalAttr:$bound); + let results = (outs Variadic:$outputs); + let regions = (region SizedRegion<1>:$body); + let hasVerifier = 1; +} +def Tcl_GraphYieldOp : Tcl_Op<"graph.yield", + [Pure, Terminator, HasParent<"GraphForOp">]> { + let arguments = (ins Variadic:$values); +} + +def Tcl_KernelOp : Tcl_Op<"kernel", [MemoryEffects<[MemRead]>, SingleBlock]> { + let arguments = (ins Variadic:$inputs, Tcl_Tactic:$tactic, + OptionalAttr:$context); + let results = (outs Tcl_Logical:$output); + let regions = (region AnyRegion:$body); + let hasVerifier = 1; +} +def Tcl_ReadOp : Tcl_Op<"read", [Pure]> { + let arguments = (ins Variadic:$inputs, + OptionalAttr:$options); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_DpeOp : Tcl_Op<"dpe", [Pure]> { + let arguments = (ins Tcl_Logical:$lhs, Tcl_Logical:$rhs); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeOp : Tcl_Op<"ve", [Pure]> { + let arguments = (ins Variadic:$inputs, Tcl_VeOpcode:$opcode); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeReduceOp : Tcl_Op<"ve_reduce", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_ReduceMode:$mode, + ArrayAttr:$axes); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_VeSelectOp : Tcl_Op<"ve_select", [Pure]> { + let arguments = (ins Tcl_Logical:$condition, Tcl_Predicate:$predicate, + AnyAttr:$threshold, AnyType:$true_value, + AnyType:$false_value); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_FilterCompactionOp : Tcl_Op<"filter_compaction", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_Logical:$condition, + Tcl_Predicate:$predicate, AnyAttr:$threshold, + ArrayAttr:$axes, AnyAttr:$max_size); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_WriteOp : Tcl_Op<"write", [Pure, Terminator]> { + let arguments = (ins Tcl_Logical:$input); + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_TD diff --git a/mlir/dialects/tcl/tcl_types.cc b/mlir/dialects/tcl/tcl_types.cc new file mode 100644 index 00000000..8b7f919a --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.cc @@ -0,0 +1,68 @@ +//===- tcl_types.cc - Furiosa TCL types -----------------------------------===// + +#include "mlir/dialects/tcl/tcl_types.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "llvm/ADT/STLExtras.h" +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +llvm::StringRef ElementName(Type t) { + if (t.isBF16()) + return "bf16"; + if (t.isF16()) + return "f16"; + if (t.isF32()) + return "f32"; + if (t.isF64()) + return "f64"; + if (isa(t)) + return "f4_e2"; + if (isa(t)) + return "f8_e4"; + if (isa(t)) + return "f8_e5"; + if (auto i = dyn_cast(t)) { + if (i.getWidth() == 1 && i.isSignless()) + return "bool"; + if (i.isUnsigned()) { + switch (i.getWidth()) { + case 8: + return "u8"; + case 16: + return "u16"; + case 32: + return "u32"; + case 64: + return "u64"; + } + } else { + switch (i.getWidth()) { + case 4: + return "i4"; + case 8: + return "i8"; + case 16: + return "i16"; + case 32: + return "i32"; + case 64: + return "i64"; + } + } + } + return {}; +} +LogicalResult LogicalType::verify(Error e, Type element, ArrayAttr axes) { + if (ElementName(element).empty()) + return e() << "unsupported TCL element type"; + return VerifyAxes(e, axes); +} +LogicalResult MappedType::verify(Error e, Type element, + DramMappingAttr mapping) { + return ElementName(element).empty() ? e() << "unsupported TCL element type" + : success(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_types.h b/mlir/dialects/tcl/tcl_types.h new file mode 100644 index 00000000..b1fdd99d --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.h @@ -0,0 +1,18 @@ +//===- tcl_types.h - Furiosa TCL types --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_H +#define MLIR_DIALECTS_TCL_TCL_TYPES_H + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/BuiltinTypes.h" +#include "llvm/ADT/StringRef.h" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.h.inc" +namespace xla::furiosa::tcl { +// Empty name denotes a type outside the SDK's 17 element types. +llvm::StringRef ElementName(mlir::Type type); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_H diff --git a/mlir/dialects/tcl/tcl_types.td b/mlir/dialects/tcl/tcl_types.td new file mode 100644 index 00000000..1cbe5e7f --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.td @@ -0,0 +1,23 @@ +//===- tcl_types.td - Furiosa TCL types --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_TD +#define MLIR_DIALECTS_TCL_TCL_TYPES_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Logical : TypeDef { + let mnemonic = "logical"; + let parameters = (ins "::mlir::Type":$elementType, "::mlir::ArrayAttr":$axes); + let assemblyFormat = "`<` $elementType `,` $axes `>`"; + let genVerifyDecl = 1; +} +def Tcl_Mapped : TypeDef { + let mnemonic = "mapped"; + let parameters = (ins "::mlir::Type":$elementType, + "::xla::furiosa::tcl::DramMappingAttr":$mapping); + let assemblyFormat = "`<` $elementType `,` $mapping `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_TD diff --git a/mlir/dialects/tcl/test.zig b/mlir/dialects/tcl/test.zig new file mode 100644 index 00000000..5adc34c0 --- /dev/null +++ b/mlir/dialects/tcl/test.zig @@ -0,0 +1,134 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const func = @import("mlir/dialects/func"); +const tcl = @import("mlir/dialects/tcl"); + +const operation_names: []const []const u8 = &.{ + "tcl.dpe", + "tcl.filter_compaction", + "tcl.graph.all_gather", + "tcl.graph.arange", + "tcl.graph.as_dram", + "tcl.graph.as_logical", + "tcl.graph.concat", + "tcl.graph.copy", + "tcl.graph.for", + "tcl.graph.full", + "tcl.graph.gather", + "tcl.graph.index_read", + "tcl.graph.index_write", + "tcl.graph.reduce_max_i32", + "tcl.graph.reshape", + "tcl.graph.scatter", + "tcl.graph.scratchpad", + "tcl.graph.slice", + "tcl.graph.sym_expr", + "tcl.graph.transmute", + "tcl.graph.vector", + "tcl.graph.yield", + "tcl.kernel", + "tcl.read", + "tcl.ve", + "tcl.ve_reduce", + "tcl.ve_select", + "tcl.write", +}; + +test { + std.testing.refAllDecls(tcl); + std.testing.refAllDecls(tcl.LogicalType); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ func, tcl }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "types and attributes verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + try std.testing.expect(t.isA(tcl.LogicalType) != null); + try std.testing.expectError(error.InvalidMlir, tcl.symbolAttr(ctx, "not an axis")); + try std.testing.expectError(error.InvalidMlir, tcl.logicalType(ctx, .float(ctx, .bf16), &.{ a, a })); + try std.testing.expectError(error.InvalidMlir, tcl.veOpcodeAttr(ctx, "unknown")); + _ = try tcl.veOpcodeAttr(ctx, "to_f16"); +} + +/// `tcl.kernel(input) { read; filter_compaction along `axis`; write }`. +fn compactionKernel(ctx: *mlir.Context, entry: *mlir.Block, axis: *const mlir.Attribute) !*mlir.Operation { + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const k = try tcl.symbolAttr(ctx, "K"); + const input = try tcl.logicalType(ctx, .int(ctx, .i32), &.{n}); + const output = try tcl.logicalType(ctx, .int(ctx, .i32), &.{k}); + const body = mlir.Block.init(&.{}, &.{}); + const x = tcl.read(ctx, &.{entry.argument(0)}, input, .{}, loc).appendTo(body); + const compacted = tcl.filter_compaction(ctx, x.result(0), x.result(0), output, .ge, .int(ctx, .i32, 0), &.{axis}, k, loc).appendTo(body); + _ = tcl.write(ctx, compacted.result(0), loc).appendTo(body); + return tcl.kernel(ctx, &.{entry.argument(0)}, output, .filter_compaction, body, .{}, loc).appendTo(entry); +} + +test "filter_compaction verifies inside a kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const entry = mlir.Block.init(&.{try tcl.logicalType(ctx, .int(ctx, .i32), &.{n})}, &.{loc}); + // The kernel verifier walks to its inputs' regions. + const function = func.func(ctx, .{ .name = "compact", .block = entry, .results = &.{}, .location = loc, .verify = false }); + defer function.deinit(); + try std.testing.expect((try compactionKernel(ctx, entry, n)).verify()); + try std.testing.expect(!(try compactionKernel(ctx, entry, try tcl.symbolAttr(ctx, "Missing"))).verify()); +} + +test "function attributes carry their tcl names" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const entry = mlir.Block.init(&.{}, &.{}); + _ = func.returns(ctx, &.{}, loc).appendTo(entry); + const function = func.func(ctx, .{ .name = "f", .block = entry, .location = loc }); + defer function.deinit(); + const config = try tcl.configAttr(ctx, &.{.named(ctx, "lowering_mode", .string(ctx, "Heuristic"))}); + tcl.annotateFunction(ctx, function, &.{.named(ctx, "A", .int(ctx, .i64, 4))}, .{ + .updated_inputs = &.{-1}, + .compiler_config = config, + .auto_config = config, + .fusible = true, + }); + try std.testing.expect(function.verify()); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{function}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "attributes {tcl.auto_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.axes = {A = 4 : i64}, tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.fusible, tcl.updated_inputs = array}") != null); +} + +test "attributes and types print and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + var type_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer type_text.deinit(); + try type_text.writer.print("{f}", .{t}); + try std.testing.expect((try mlir.Type.parse(ctx, type_text.written())).eql(t)); + var attr_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer attr_text.deinit(); + try attr_text.writer.print("{f}", .{a}); + try std.testing.expect((try mlir.Attribute.parse(ctx, attr_text.written())).eql(a)); +} diff --git a/mlir/dialects/tcl/ve_instructions.inc b/mlir/dialects/tcl/ve_instructions.inc new file mode 100644 index 00000000..657eddca --- /dev/null +++ b/mlir/dialects/tcl/ve_instructions.inc @@ -0,0 +1,48 @@ +//===- ve_instructions.inc - Furiosa TCL VE instructions ------------------===// +// +// SDK 2026.3 VE opcode vocabulary; conversion widths 0..31 are parametric. + +VE("exp", "exp", 1) +VE("neg_exp", "neg_exp", 1) +VE("sqrt", "sqrt", 1) +VE("tanh", "tanh", 1) +VE("sigmoid", "sigmoid", 1) +VE("erf", "erf", 1) +VE("log", "log", 1) +VE("sin", "sin", 1) +VE("cos", "cos", 1) +VE("negf", "neg_f", 1) +VE("absf", "abs_f", 1) +VE("addi", "+_fxp", 2) +VE("addi_sat", "+_fxp_sat", 2) +VE("subi", "-_fxp", 2) +VE("subi_sat", "-_fxp_sat", 2) +VE("mul_q31", "*_fxp", 2) +VE("muli", "*_i", 2) +VE("shl", "<<", 2) +VE("shl_sat", "<<_sat", 2) +VE("shr_logical", ">>>", 2) +VE("shr_arith", ">>", 2) +VE("shr_arith_round", ">>_round", 2) +VE("bit_andi", "&_i", 2) +VE("bit_ori", "|_i", 2) +VE("bit_xori", "^_i", 2) +VE("mini", "min_i", 2) +VE("maxi", "max_i", 2) +VE("absmini", "|min|_i", 2) +VE("absmaxi", "|max|_i", 2) +VE("addf", "+", 2) +VE("subf", "-", 2) +VE("mulf", "*", 2) +VE("divf", "/", 2) +VE("bit_andf", "&_f", 2) +VE("bit_orf", "|_f", 2) +VE("bit_xorf", "^_f", 2) +VE("minf", "min_f", 2) +VE("maxf", "max_f", 2) +VE("absminf", "|min|_f", 2) +VE("absmaxf", "|max|_f", 2) +VE("reinterpret_as_float", "as_f", 1) +VE("reinterpret_as_int", "as_i", 1) +VE("to_float", "to_f31", 1) +VE("to_int", "to_i31", 1) diff --git a/mlir/dialects/ttir/BUILD.bazel b/mlir/dialects/ttir/BUILD.bazel new file mode 100644 index 00000000..8ab5598f --- /dev/null +++ b/mlir/dialects/ttir/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@triton//:TritonDialects", + ], +) + +zig_library( + name = "ttir", + import_name = "mlir/dialects/ttir", + main = "ttir.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":ttir", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/scf", + ], +) diff --git a/mlir/dialects/ttir/capi.cc b/mlir/dialects/ttir/capi.cc new file mode 100644 index 00000000..0e7afb28 --- /dev/null +++ b/mlir/dialects/ttir/capi.cc @@ -0,0 +1,167 @@ +//===- capi.cc - C API for the Triton tt dialect --------------------------===// + +#include "capi.h" + +#include "triton/Dialect/Triton/IR/Dialect.h" +#include "triton/Dialect/Triton/IR/Types.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Note: the second arg `tt` controls the generated symbol name +// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on +// the Zig side resolves at comptime (see `DialectHandle.fromString` in +// mlir/mlir.zig). Must match the dialect's declared name (`let name = "tt"` in +// TritonDialect.td). +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) + +static mlir::MLIRContext *loadTriton(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +using mlir::cast; +using mlir::isa; + +MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t addressSpace) { + return wrap(mlir::triton::PointerType::get( + unwrap(pointee), static_cast(addressSpace))); +} + +MlirType mlirTritonPointerTypeGetPointee(MlirType type) { + return wrap(cast(unwrap(type)).getPointeeType()); +} + +int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type) { + return static_cast( + cast(unwrap(type)).getAddressSpace()); +} + +bool mlirTypeIsATritonPointer(MlirType type) { + return isa(unwrap(type)); +} + +MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, + MlirAttribute sharedLayout) { + mlir::Attribute layoutAttr; + if (!mlirAttributeIsNull(sharedLayout)) { + layoutAttr = unwrap(sharedLayout); + } + // TensorDescType::get is a TypeBuilderWithInferredContext: the context is + // inferred from elementType. Signature: (shape, elementType, sharedLayout). + return wrap(mlir::triton::TensorDescType::get( + llvm::ArrayRef(shape, rank), unwrap(elementType), layoutAttr)); +} + +bool mlirTypeIsATritonTensorDesc(MlirType type) { + return isa(unwrap(type)); +} + +MlirAttribute mlirTritonProgramIDDimAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ProgramIDDimAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonCacheModifierAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::CacheModifierAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonEvictionPolicyAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::EvictionPolicyAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPaddingOptionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PaddingOptionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRoundingModeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RoundingModeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonInputPrecisionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::InputPrecisionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RMWOpAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSemanticAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSyncScopeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSyncScopeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPropagateNanAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PropagateNanAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonScaleDotElemTypeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ScaleDotElemTypeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonDescriptorReduceKindAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::DescriptorReduceKindAttr::get( + ctx, static_cast(value))); +} diff --git a/mlir/dialects/ttir/capi.h b/mlir/dialects/ttir/capi.h new file mode 100644 index 00000000..efd0aae9 --- /dev/null +++ b/mlir/dialects/ttir/capi.h @@ -0,0 +1,59 @@ +//===- capi.h - C API for the Triton tt dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TTIR_CAPI_H +#define MLIR_DIALECTS_TTIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); + +// !tt.ptr +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGet(MlirType pointee, + int32_t addressSpace); +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGetPointee(MlirType type); +MLIR_CAPI_EXPORTED int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonPointer(MlirType type); + +// !tt.tensordesc> — result of tt.make_tensor_descriptor. +// `shape` is a `rank`-element array of int64_t block dims; `elementType` is +// the scalar element type; `sharedLayout` may be a null MlirAttribute for none. +MLIR_CAPI_EXPORTED MlirType +mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, MlirAttribute sharedLayout); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonTensorDesc(MlirType type); + +// I32EnumAttr constructors +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonProgramIDDimAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonCacheModifierAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonEvictionPolicyAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPaddingOptionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonRoundingModeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonInputPrecisionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSyncScopeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPropagateNanAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonScaleDotElemTypeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonDescriptorReduceKindAttrGet(MlirContext context, int32_t value); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TTIR_CAPI_H diff --git a/mlir/dialects/ttir/test.zig b/mlir/dialects/ttir/test.zig new file mode 100644 index 00000000..04361e4e --- /dev/null +++ b/mlir/dialects/ttir/test.zig @@ -0,0 +1,172 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); + +const operation_names: []const []const u8 = &.{ + "tt.addptr", + "tt.assert", + "tt.atomic_cas", + "tt.atomic_rmw", + "tt.bitcast", + "tt.broadcast", + "tt.call", + "tt.clampf", + "tt.descriptor_gather", + "tt.descriptor_load", + "tt.descriptor_reduce", + "tt.descriptor_scatter", + "tt.descriptor_store", + "tt.dot", + "tt.dot_scaled", + "tt.elementwise_inline_asm", + "tt.expand_dims", + "tt.extern_elementwise", + "tt.fp_to_fp", + "tt.func", + "tt.gather", + "tt.get_num_programs", + "tt.get_program_id", + "tt.histogram", + "tt.int_to_ptr", + "tt.join", + "tt.load", + "tt.make_range", + "tt.make_tensor_descriptor", + "tt.map_elementwise", + "tt.map_elementwise.return", + "tt.mulhiui", + "tt.precise_divf", + "tt.precise_sqrt", + "tt.print", + "tt.ptr_to_int", + "tt.reduce", + "tt.reduce.return", + "tt.reshape", + "tt.return", + "tt.scan", + "tt.scan.return", + "tt.splat", + "tt.split", + "tt.store", + "tt.trans", + "tt.unsplat", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ttir); + inline for (comptime std.meta.declarations(ttir)) |decl| { + const T = @field(ttir, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "tt.func round-trip" { + // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } + // by constructing it with the Zig wrappers, printing to string, and re-parsing. + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + ttir.registerDialect(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + try std.testing.expect(ctx.isRegisteredOperation("tt.func")); + try std.testing.expect(ctx.isRegisteredOperation("tt.return")); + try std.testing.expect(ctx.isRegisteredOperation("tt.load")); + try std.testing.expect(ctx.isRegisteredOperation("tt.store")); + try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); + try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); + try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); + try std.testing.expect(ctx.isRegisteredOperation("scf.for")); + try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); + + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_ty = ttir.pointerType(f32_ty, 1); + + const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); + + const arg0 = entry.argument(0); + const arg1 = entry.argument(1); + + // %0 = tt.load %arg0 : !tt.ptr + const loaded = ttir.load(ctx, arg0, f32_ty, .none, .normal, false, .{}, loc); + _ = loaded.appendTo(entry); + + // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import + const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); + const one = mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{f32_ty} }, + .attributes = &.{.named(ctx, "value", one_attr)}, + .location = loc, + }); + _ = one.appendTo(entry); + + // %2 = arith.addf %0, %1 : f32 + const summed = mlir.Operation.make(ctx, "arith.addf", .{ + .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, + .result_type_inference = true, + .location = loc, + }); + _ = summed.appendTo(entry); + + // tt.store %arg1, %2 + _ = ttir.store(ctx, arg1, summed.result(0), .none, .normal, .{}, loc).appendTo(entry); + + // tt.return + _ = ttir.return_(ctx, &.{}, loc).appendTo(entry); + + const f = ttir.func(ctx, .{ + .name = "add_one", + .block = entry, + .location = loc, + }); + _ = f.appendTo(module.body()); + + // Verify the operation. + try std.testing.expect(module.operation().verify()); + + // Print IR and re-parse it to confirm it's valid text. + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module.operation()}); + + const parsed = try mlir.Module.parse(ctx, al.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "divisibility is an i32 attribute of the tt dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ttir.divisibility(ctx, 16)})}); + try std.testing.expectEqualStrings("{tt.divisibility = 16 : i32}", w.buffered()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ttir.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig new file mode 100644 index 00000000..ee7b5307 --- /dev/null +++ b/mlir/dialects/ttir/ttir.zig @@ -0,0 +1,1231 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tt"); +} + +// ============================================================================= +// Types — !tt.ptr +// ============================================================================= + +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsATritonPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + pointee_type: *const mlir.Type, + address_space: i32, + }; + + pub fn get(args: InitArgs) *const PointerType { + return @ptrCast(c.mlirTritonPointerTypeGet(args.pointee_type.ptr(), args.address_space).ptr); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return @ptrCast(c.mlirTritonPointerTypeGetPointee(self.ptr()).ptr); + } + + pub fn addressSpace(self: *const PointerType) i32 { + return c.mlirTritonPointerTypeGetAddressSpace(self.ptr()); + } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// Convenience alias: `!tt.ptr` as a plain *const mlir.Type. +pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const mlir.Type { + return @ptrCast(PointerType.get(.{ .pointee_type = pointee_type, .address_space = address_space })); +} + +// ----------------------------------------------------------------------------- +// !tt.tensordesc> — TMA descriptor type. +// ----------------------------------------------------------------------------- + +pub const TensorDescType = opaque { + const M = mlir.Methods(TensorDescType, c.MlirType); + + pub const isAFn = c.mlirTypeIsATritonTensorDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: []const i64, + element_type: *const mlir.Type, + /// May be null for an unlayout'd descriptor. + shared_layout: ?*const mlir.Attribute = null, + }; + + pub fn get(args: InitArgs) *const TensorDescType { + const attr_ptr = if (args.shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; + return @ptrCast(c.mlirTritonTensorDescTypeGet( + @intCast(args.shape.len), + args.shape.ptr, + args.element_type.ptr(), + attr_ptr, + ).ptr); + } + + pub fn type_(self: *const TensorDescType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tensorDescType( + shape: []const i64, + element_type: *const mlir.Type, + opts: struct { + shared_layout: ?*const mlir.Attribute = null, + }, +) *const mlir.Type { + return @ptrCast(TensorDescType.get(.{ .shape = shape, .element_type = element_type, .shared_layout = opts.shared_layout })); +} + +// ============================================================================= +// Enum attributes +// ============================================================================= + +pub const ProgramIDDim = enum(i32) { x = 0, y = 1, z = 2 }; +pub const CacheModifier = enum(i32) { none = 1, ca = 2, cg = 3, wb = 4, cs = 5, wt = 6, cv = 7 }; +pub const EvictionPolicy = enum(i32) { normal = 1, evict_first = 2, evict_last = 3 }; +pub const PaddingOption = enum(i32) { pad_zero = 1, pad_nan = 2 }; +pub const RoundingMode = enum(i32) { rtz = 0, rtne = 1 }; +pub const InputPrecision = enum(i32) { tf32 = 0, tf32x3 = 1, ieee = 2, bf16x3 = 3, bf16x6 = 4 }; + +/// Atomic read-modify-write operations. +pub const RMWOp = enum(i32) { + @"and" = 1, + @"or" = 2, + xor = 3, + add = 4, + fadd = 5, + max = 6, + min = 7, + umax = 8, + umin = 9, + xchg = 10, +}; + +/// Memory ordering semantics. +pub const MemSemantic = enum(i32) { + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, +}; + +/// Memory sync scope. +pub const MemSyncScope = enum(i32) { + gpu = 1, + cta = 2, + sys = 3, +}; + +/// NaN propagation for clampf. 0xFFFF is the exact .td encoding. +pub const PropagateNan = enum(i32) { + none = 0, + all = 0xFFFF, +}; + +/// Packed FP element types for tt.dot_scaled. +pub const ScaleDotElemType = enum(i32) { + e4m3 = 0, + e5m2 = 1, + e2m3 = 2, + e3m2 = 3, + e2m1 = 4, + bf16 = 5, + fp16 = 6, +}; + +/// Reduction kind for tt.descriptor_reduce. +pub const DescriptorReduceKind = enum(i32) { + add = 1, + min = 2, + max = 3, + inc = 4, + dec = 5, + @"and" = 6, + @"or" = 7, + xor = 8, +}; + +pub fn programDimAttr(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { + return @ptrCast(c.mlirTritonProgramIDDimAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn cacheModifierAttr(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { + return @ptrCast(c.mlirTritonCacheModifierAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn evictionPolicyAttr(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { + return @ptrCast(c.mlirTritonEvictionPolicyAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn paddingOptionAttr(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPaddingOptionAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn roundingModeAttr(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRoundingModeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn inputPrecisionAttr(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { + return @ptrCast(c.mlirTritonInputPrecisionAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn rmwOpAttr(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRMWOpAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn memSemanticAttr(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSemanticAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn memSyncScopeAttr(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSyncScopeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn propagateNanAttr(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPropagateNanAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn scaleDotElemTypeAttr(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { + return @ptrCast(c.mlirTritonScaleDotElemTypeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn descriptorReduceKindAttr(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { + return @ptrCast(c.mlirTritonDescriptorReduceKindAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +/// `tt.divisibility`: an argument attribute promising that the argument (the +/// address, for a pointer) is a multiple of `divisor`. +pub fn divisibility(ctx: *mlir.Context, divisor: u32) mlir.NamedAttribute { + return .named(ctx, "tt.divisibility", .int(ctx, .i32, divisor)); +} + +// ============================================================================= +// Function ops — tt.func / tt.return +// ============================================================================= + +/// Args for building a `tt.func` op. +pub const FuncOpArgs = struct { + pub const Visibility = enum { public, private }; + + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, + /// `tt.func` has `HasParent<"ModuleOp">`, so verification before appending + /// to a module always fails. Defer to the module-level verify instead. + verify: bool = false, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", .typeAttr(.function( + ctx, + args.args orelse args_: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args_ args_buffer.constSlice(); + }, + args.results orelse results_: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results_ results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); + } + // Always emit noinline even when false — matches Triton's IR convention. + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", .boolean(ctx, args.no_inline))); + + return .make(ctx, "tt.func", .{ + .blocks = &.{args.block}, + .attributes = attr_tuples_buffer.constSlice(), + .location = args.location, + .verify = args.verify, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// SPMD ops +// ============================================================================= + +pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.get_program_id", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDimAttr(ctx, axis)), + }, + .location = location, + }); +} + +pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.get_num_programs", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDimAttr(ctx, axis)), + }, + .location = location, + }); +} + +// ============================================================================= +// Range +// ============================================================================= + +pub fn make_range( + ctx: *mlir.Context, + start: i32, + end: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.make_range", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "start", .int(ctx, .i32, start)), + .named(ctx, "end", .int(ctx, .i32, end)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +pub fn splat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.splat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn expand_dims( + ctx: *mlir.Context, + src: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.expand_dims", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + allow_reorder: bool, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (allow_reorder) { + attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", .unit(ctx))); + } + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); + } + return .make(ctx, "tt.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn trans( + ctx: *mlir.Context, + src: *const mlir.Value, + order: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.trans", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "order", .denseArray(ctx, .i32, order)), + }, + .location = location, + }); +} + +// ============================================================================= +// Pointer arithmetic & memory +// ============================================================================= + +/// tt.addptr — result type matches ptr type (TypesMatchWith in TritonOps.td). +pub fn addptr( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + offset: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.addptr", .{ + .operands = .{ .flat = &.{ ptr_val, offset } }, + .results = .{ .flat = &.{ptr_val.type_()} }, + .location = location, + }); +} + +/// tt.load — AttrSizedOperandSegments: (ptr, [mask], [other]). Uses flat operand list + +/// explicit operandSegmentSizes because Operation.make variadic coercion zeros mixed-length segments. +pub fn load( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + result_type: *const mlir.Type, + cache: CacheModifier, + evict: EvictionPolicy, + is_volatile: bool, + opts: struct { + mask: ?*const mlir.Value = null, + other: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendAssumeCapacity(ptr_val); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + const other_len: i32 = if (opts.other) |o| blk: { + operands_buf.appendAssumeCapacity(o); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, mask_len, other_len }; + return .make(ctx, "tt.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + .named(ctx, "isVolatile", .boolean(ctx, is_volatile)), + }, + .location = location, + }); +} + +/// tt.store — operands: (ptr, value, [mask]). Flat operand list; mask is trailing variadic. +pub fn store( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + value: *const mlir.Value, + cache: CacheModifier, + evict: EvictionPolicy, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); + if (opts.mask) |m| buf.appendAssumeCapacity(m); + return .make(ctx, "tt.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute +// ============================================================================= + +pub fn dot( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + result_type: *const mlir.Type, + input_precision_: InputPrecision, + max_num_imprecise_acc: i32, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.dot", .{ + .operands = .{ .flat = &.{ a, b, c_acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "inputPrecision", inputPrecisionAttr(ctx, input_precision_)), + .named(ctx, "maxNumImpreciseAcc", .int(ctx, .i32, max_num_imprecise_acc)), + }, + .location = location, + }); +} + +/// tt.reduce — combine_block must have args [elem, elem] matching srcs' element types, +/// terminated with tt.reduce.return. +pub fn reduce( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // Single variadic operand group — flat list, no operandSegmentSizes needed. + return .make(ctx, "tt.reduce", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn reduce_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.reduce.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Elementwise casts specific to Triton (arith.bitcast doesn't support ptrs) +// ============================================================================= + +/// tt.int_to_ptr — cast int-like to ptr-like (element-wise, same shape). +pub fn int_to_ptr( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.int_to_ptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.ptr_to_int — cast ptr-like to int-like (element-wise, same shape). +pub fn ptr_to_int( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.ptr_to_int", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.bitcast — cast between types of the same bit-width (also works on ptrs, +/// unlike arith.bitcast). +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.fp_to_fp — float to float cast, optionally with a rounding mode. +/// Pass `rounding = null` to omit the rounding attribute. +pub fn fp_to_fp( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingModeAttr(ctx, rm))); + } + return .make(ctx, "tt.fp_to_fp", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Math ops +// ============================================================================= + +/// tt.clampf — clamp float tensor to [min, max] element-wise. +pub fn clampf( + ctx: *mlir.Context, + x: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + propagate_nan: PropagateNan, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.clampf", .{ + .operands = .{ .flat = &.{ x, min, max } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = &.{ + .named(ctx, "propagateNan", propagateNanAttr(ctx, propagate_nan)), + }, + .location = location, + }); +} + +/// tt.precise_sqrt — precise sqrt (SameOperandsAndResultType). +pub fn precise_sqrt( + ctx: *mlir.Context, + x: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.precise_sqrt", .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.precise_divf — precise float div (SameOperandsAndResultType). +pub fn precise_divf( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.precise_divf", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.mulhiui — most significant N bits of 2N-bit unsigned product. +pub fn mulhiui( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.mulhiui", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation — more +// ============================================================================= + +/// tt.unsplat — tensor-of-1-element → scalar. +pub fn unsplat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.unsplat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.join — join two tensors along a new minor dimension (shape gains +1 dim of size 2). +pub fn join( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.join", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.split — split a tensor along its last dimension (size must be 2). Returns (lhs, rhs). +pub fn split( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.split", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Gather / Histogram +// ============================================================================= + +/// tt.gather — gather elements from `src` using `indices` along `axis`. +/// Result shape matches `indices` shape. +pub fn gather( + ctx: *mlir.Context, + src: *const mlir.Value, + indices: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "axis", .int(ctx, .i32, axis))); + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); + } + return .make(ctx, "tt.gather", .{ + .operands = .{ .flat = &.{ src, indices } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// tt.histogram — build a histogram of `src`. Output size = number of bins. +pub fn histogram( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + buf.appendAssumeCapacity(src); + if (opts.mask) |m| buf.appendAssumeCapacity(m); + return .make(ctx, "tt.histogram", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Scan — parallel scan with combine region (like reduce) +// ============================================================================= + +/// tt.scan — combine_block must already contain the tt.scan.return terminator +/// and have arguments with element types matching `srcs`. +pub fn scan( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + reverse: bool, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.scan", .{ + .operands = .{ .variadic = &.{srcs} }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + .named(ctx, "reverse", .boolean(ctx, reverse)), + }, + .location = location, + }); +} + +pub fn scan_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.scan.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Atomics +// ============================================================================= + +/// tt.atomic_rmw — (ptr, val, [mask]). Result type matches val's type. +pub fn atomic_rmw( + ctx: *mlir.Context, + rmw: RMWOp, + ptr_val: *const mlir.Value, + val: *const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return .make(ctx, "tt.atomic_rmw", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "atomic_rmw_op", rmwOpAttr(ctx, rmw)), + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), + }, + .location = location, + }); +} + +/// tt.atomic_cas — compare-and-swap. Result type matches val's type. +pub fn atomic_cas( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + cmp: *const mlir.Value, + val: *const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.atomic_cas", .{ + .operands = .{ .flat = &.{ ptr_val, cmp, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), + }, + .location = location, + }); +} + +// ============================================================================= +// Debug +// ============================================================================= + +/// tt.assert — device-side assert. `condition` must be i1 or tensor<...xi1>. +pub fn assert_( + ctx: *mlir.Context, + condition: *const mlir.Value, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.assert", .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{ + .named(ctx, "message", .string(ctx, message)), + }, + .location = location, + }); +} + +// ============================================================================= +// Call +// ============================================================================= + +/// tt.call — direct call to another tt.func in the same module. +/// `opts.arg_attrs` / `opts.res_attrs` propagate hints like tt.divisibility; +/// leave them unset to omit them. +pub fn call( + ctx: *mlir.Context, + callee: []const u8, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + opts: struct { + arg_attrs: ?[]const *const mlir.Attribute = null, + res_attrs: ?[]const *const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "callee", .flatSymbolRef(ctx, callee))); + if (opts.arg_attrs) |aa| { + attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, aa))); + } + if (opts.res_attrs) |ra| { + attrs.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, ra))); + } + return .make(ctx, "tt.call", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Scaled dot (microscaling spec). +// ============================================================================= + +/// tt.dot_scaled — dot with optional per-block scale factors. +/// a_scale/b_scale may be null. AttrSizedOperandSegments: (a, b, c, [a_scale], [b_scale]). +pub fn dot_scaled( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + a_elem_type: ScaleDotElemType, + b_elem_type: ScaleDotElemType, + result_type: *const mlir.Type, + fast_math: bool, + lhs_k_pack: bool, + rhs_k_pack: bool, + opts: struct { + a_scale: ?*const mlir.Value = null, + b_scale: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + buf.appendSliceAssumeCapacity(&.{ a, b, c_acc }); + const a_scale_len: i32 = if (opts.a_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + const b_scale_len: i32 = if (opts.b_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, a_scale_len, b_scale_len }; + return .make(ctx, "tt.dot_scaled", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "a_elem_type", scaleDotElemTypeAttr(ctx, a_elem_type)), + .named(ctx, "b_elem_type", scaleDotElemTypeAttr(ctx, b_elem_type)), + .named(ctx, "fastMath", .boolean(ctx, fast_math)), + .named(ctx, "lhs_k_pack", .boolean(ctx, lhs_k_pack)), + .named(ctx, "rhs_k_pack", .boolean(ctx, rhs_k_pack)), + }, + .location = location, + }); +} + +// ============================================================================= +// Extern elementwise (call library function pointwise) +// ============================================================================= + +pub fn extern_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + result_type: *const mlir.Type, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + pure: bool, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.extern_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "libname", .string(ctx, libname)), + .named(ctx, "libpath", .string(ctx, libpath)), + .named(ctx, "symbol", .string(ctx, symbol)), + .named(ctx, "pure", .boolean(ctx, pure)), + }, + .location = location, + }); +} + +// ============================================================================= +// Device-side print +// ============================================================================= + +/// tt.print — device-side print. `is_signed` must be `args.len` long and +/// indicates signedness per arg (1 = signed, 0 = unsigned) for printf-style +/// integer formatting. +pub fn print( + ctx: *mlir.Context, + prefix: []const u8, + hex: bool, + args: []const *const mlir.Value, + is_signed: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(args.len == is_signed.len); + return .make(ctx, "tt.print", .{ + .operands = .{ .flat = args }, + .attributes = &.{ + .named(ctx, "prefix", .string(ctx, prefix)), + .named(ctx, "hex", .boolean(ctx, hex)), + .named(ctx, "isSigned", .denseArray(ctx, .i32, is_signed)), + }, + .location = location, + }); +} + +// ============================================================================= +// Tensor descriptors (TMA) +// ============================================================================= + +/// tt.make_tensor_descriptor — build a TMA descriptor from a base ptr, runtime +/// shape/strides, and the compile-time block shape baked into `result_type`. +pub fn make_tensor_descriptor( + ctx: *mlir.Context, + base: *const mlir.Value, + shape: []const *const mlir.Value, + strides: []const *const mlir.Value, + padding: PaddingOption, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(base); + buf.appendSliceAssumeCapacity(shape); + buf.appendSliceAssumeCapacity(strides); + // SameVariadicOperandSize — flatten all operands. + return .make(ctx, "tt.make_tensor_descriptor", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "padding", paddingOptionAttr(ctx, padding)), + }, + .location = location, + }); +} + +pub fn descriptor_load( + ctx: *mlir.Context, + desc: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + cache: CacheModifier, + evict: EvictionPolicy, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(desc); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_load", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + }, + .location = location, + }); +} + +pub fn descriptor_store( + ctx: *mlir.Context, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_store", .{ + .operands = .{ .flat = buf.constSlice() }, + .location = location, + }); +} + +pub fn descriptor_reduce( + ctx: *mlir.Context, + kind: DescriptorReduceKind, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_reduce", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "kind", descriptorReduceKindAttr(ctx, kind)), + }, + .location = location, + }); +} + +pub fn descriptor_gather( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.descriptor_gather", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn descriptor_scatter( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.descriptor_scatter", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset, src } }, + .location = location, + }); +} + +// ============================================================================= +// Map elementwise — like a pointwise `map` with a user-defined scalar region. +// ============================================================================= + +/// tt.map_elementwise — `scalar_block` receives `pack * srcs.len` scalar args +/// (packed groups of `pack` elements per src tensor) and must be terminated +/// with `tt.map_elementwise.return`. +pub fn map_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + pack: i32, + scalar_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.map_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{scalar_block}, + .attributes = &.{ + .named(ctx, "pack", .int(ctx, .i32, pack)), + }, + .location = location, + }); +} + +pub fn map_elementwise_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.map_elementwise.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Inline assembly +// ============================================================================= + +/// tt.elementwise_inline_asm — pointwise inline asm returning one or more +/// tensors. +pub fn elementwise_inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + constraints: []const u8, + args: []const *const mlir.Value, + result_types: []const *const mlir.Type, + pure: bool, + packed_element: i32, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.elementwise_inline_asm", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "asm_string", .string(ctx, asm_string)), + .named(ctx, "constraints", .string(ctx, constraints)), + .named(ctx, "pure", .boolean(ctx, pure)), + .named(ctx, "packed_element", .int(ctx, .i32, packed_element)), + }, + .location = location, + }); +} diff --git a/mlir/dialects/ub/BUILD.bazel b/mlir/dialects/ub/BUILD.bazel new file mode 100644 index 00000000..039ca5ff --- /dev/null +++ b/mlir/dialects/ub/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:UBDialect", + ], +) + +zig_library( + name = "ub", + import_name = "mlir/dialects/ub", + main = "ub.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":ub", + "//mlir", + ], +) diff --git a/mlir/dialects/ub/capi.cc b/mlir/dialects/ub/capi.cc new file mode 100644 index 00000000..cf37a2c0 --- /dev/null +++ b/mlir/dialects/ub/capi.cc @@ -0,0 +1,8 @@ +//===- capi.cc - C API for the ub dialect ---------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) diff --git a/mlir/dialects/ub/capi.h b/mlir/dialects/ub/capi.h new file mode 100644 index 00000000..76765ad9 --- /dev/null +++ b/mlir/dialects/ub/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the ub dialect --------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_UB_CAPI_H +#define MLIR_DIALECTS_UB_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_UB_CAPI_H diff --git a/mlir/dialects/ub/test.zig b/mlir/dialects/ub/test.zig new file mode 100644 index 00000000..738d1378 --- /dev/null +++ b/mlir/dialects/ub/test.zig @@ -0,0 +1,23 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ub = @import("mlir/dialects/ub"); + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ub); + inline for (comptime std.meta.declarations(ub)) |decl| { + const T = @field(ub, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "registerDialect registers ub" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ub.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} diff --git a/mlir/dialects/ub/ub.zig b/mlir/dialects/ub/ub.zig new file mode 100644 index 00000000..b5352257 --- /dev/null +++ b/mlir/dialects/ub/ub.zig @@ -0,0 +1,5 @@ +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("ub"); +} diff --git a/mlir/dialects/vector/BUILD.bazel b/mlir/dialects/vector/BUILD.bazel new file mode 100644 index 00000000..579fa599 --- /dev/null +++ b/mlir/dialects/vector/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:VectorDialect", + ], +) + +zig_library( + name = "vector", + import_name = "mlir/dialects/vector", + main = "vector.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIVector", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":vector", + "//mlir", + ], +) diff --git a/mlir/dialects/vector/capi.cc b/mlir/dialects/vector/capi.cc new file mode 100644 index 00000000..f22ad655 --- /dev/null +++ b/mlir/dialects/vector/capi.cc @@ -0,0 +1,51 @@ +//===- capi.cc - C API for the vector dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +static mlir::MLIRContext *loadVector(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional kind = + mlir::vector::symbolizeCombiningKind(value); + if (!ctx || !kind) + return {nullptr}; + return wrap(mlir::vector::CombiningKindAttr::get(ctx, *kind)); +} + +uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional type = + mlir::vector::symbolizeIteratorType(value); + if (!ctx || !type) + return {nullptr}; + return wrap(mlir::vector::IteratorTypeAttr::get(ctx, *type)); +} + +uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/vector/capi.h b/mlir/dialects/vector/capi.h new file mode 100644 index 00000000..75e12cbd --- /dev/null +++ b/mlir/dialects/vector/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the vector dialect ----------------------*- C -*-===// +// +// C entry points for the `vector` dialect's enum attributes, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_VECTOR_CAPI_H +#define MLIR_DIALECTS_VECTOR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #vector.kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorCombiningKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); + +// #vector.iterator_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorIteratorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/mlir/dialects/vector/test.zig b/mlir/dialects/vector/test.zig new file mode 100644 index 00000000..ca737ae7 --- /dev/null +++ b/mlir/dialects/vector/test.zig @@ -0,0 +1,86 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +const operation_names: []const []const u8 = &.{ + "vector.bitcast", + "vector.broadcast", + "vector.compressstore", + "vector.constant_mask", + "vector.contract", + "vector.create_mask", + "vector.deinterleave", + "vector.expandload", + "vector.extract", + "vector.extract_strided_slice", + "vector.fma", + "vector.from_elements", + "vector.gather", + "vector.insert", + "vector.insert_strided_slice", + "vector.interleave", + "vector.load", + "vector.maskedload", + "vector.maskedstore", + "vector.multi_reduction", + "vector.reduction", + "vector.scatter", + "vector.shape_cast", + "vector.shuffle", + "vector.step", + "vector.store", + "vector.to_elements", + "vector.transfer_read", + "vector.transfer_write", + "vector.transpose", + "vector.type_cast", + "vector.vscale", + "vector.yield", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(vector); + inline for (comptime std.meta.declarations(vector)) |decl| { + const T = @field(vector, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "vector enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(vector.CombiningKind).@"enum".field_names) |name| { + const value: vector.CombiningKind = @field(vector.CombiningKind, name); + const attr = try vector.CombiningKindAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(value.attribute(ctx).isA(vector.CombiningKindAttr) != null); + try expectPrints("#vector.kind<" ++ name ++ ">", attr); + } + inline for (@typeInfo(vector.IteratorType).@"enum".field_names) |name| { + const value: vector.IteratorType = @field(vector.IteratorType, name); + const attr = try vector.IteratorTypeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + vector.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/vector/vector.zig b/mlir/dialects/vector/vector.zig new file mode 100644 index 00000000..1696d04b --- /dev/null +++ b/mlir/dialects/vector/vector.zig @@ -0,0 +1,820 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("vector"); +} + +// ============================================================================= +// Enum attributes +// ============================================================================= + +/// Kind of combining function for contractions and reductions. +pub const CombiningKind = enum(u32) { + add = 0, + mul = 1, + minui = 2, + minsi = 3, + minnumf = 4, + maxui = 5, + maxsi = 6, + maxnumf = 7, + @"and" = 8, + @"or" = 9, + xor = 10, + minimumf = 11, + maximumf = 12, + minimumnumf = 13, + maximumnumf = 14, + + pub fn attribute(self: CombiningKind, ctx: *mlir.Context) *const mlir.Attribute { + return (CombiningKindAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); + } +}; + +/// Iterator type of a `vector.contract` dimension. +pub const IteratorType = enum(u32) { + parallel = 0, + reduction = 1, + + pub fn attribute(self: IteratorType, ctx: *mlir.Context) *const mlir.Attribute { + return (IteratorTypeAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); + } +}; + +/// `#vector.kind<...>`. +pub const CombiningKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorCombiningKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CombiningKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CombiningKind { + return @fromBackingInt(@intCast(c.mlirVectorCombiningKindAttrGetValue(self.ptr()))); + } +}; + +/// `#vector.iterator_type<...>`. +pub const IteratorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorIteratorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "iterator_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IteratorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IteratorType { + return @fromBackingInt(@intCast(c.mlirVectorIteratorTypeAttrGetValue(self.ptr()))); + } +}; + +// ============================================================================= +// Element / lane access — extract / insert / from_elements / to_elements +// ============================================================================= + +/// vector.extract — pull a single element or sub-vector out of `src`. Static +/// position entries equal to `kPoison` (== `INT64_MIN` per VectorOps.td) come +/// from `dynamic_position`. The result type is provided by the caller. +pub const kPoison: i64 = std.math.minInt(i64); + +pub fn extract( + ctx: *mlir.Context, + src: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(src); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.extract", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.insert — write `value` into `dest` at the given position. The result +/// has the same type as `dest`. +pub fn insert( + ctx: *mlir.Context, + value: *const mlir.Value, + dest: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, dest }); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.insert", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.from_elements — build a vector from an exact-size list of scalars. +pub fn from_elements( + ctx: *mlir.Context, + elements: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.from_elements", .{ + .operands = .{ .flat = elements }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.to_elements — destructure a vector into its scalar lanes. +pub fn to_elements( + ctx: *mlir.Context, + src: *const mlir.Value, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.to_elements", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +// ============================================================================= +// Strided slice — extract / insert with offsets/sizes/strides +// ============================================================================= + +pub const StridedSliceArgs = struct { + offsets: []const i64, + sizes: []const i64, + strides: []const i64, +}; + +pub fn extract_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + args: StridedSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.extract_strided_slice", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "offsets", .intArray(ctx, i64, args.offsets)), + .named(ctx, "sizes", .intArray(ctx, i64, args.sizes)), + .named(ctx, "strides", .intArray(ctx, i64, args.strides)), + }, + .location = location, + }); +} + +pub fn insert_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + dest: *const mlir.Value, + offsets: []const i64, + strides: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.insert_strided_slice", .{ + .operands = .{ .flat = &.{ src, dest } }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "offsets", .intArray(ctx, i64, offsets)), + .named(ctx, "strides", .intArray(ctx, i64, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +/// vector.broadcast — broadcast a scalar / lower-rank vector to a wider one. +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.shape_cast — reshape a vector preserving total number of lanes. +pub fn shape_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shape_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.bitcast — same total bits, possibly different element type / shape. +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.type_cast — cast a `memref>` to `memref<...>` (and vice- +/// versa). Used in vector lowering / unrolling. +pub fn type_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.type_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.transpose — permute dims of a vector. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Lane shuffle — shuffle / interleave / deinterleave +// ============================================================================= + +/// vector.shuffle — concat lhs and rhs into a logical vector (rank 1) and +/// pick `mask`-listed lanes to form the result. +pub fn shuffle( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + mask: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shuffle", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask", .denseArray(ctx, .i64, mask)), + }, + .location = location, + }); +} + +/// vector.interleave — interleave even/odd lanes from lhs/rhs. +pub fn interleave( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.interleave", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.deinterleave — split a vector into (even-lanes, odd-lanes). +pub fn deinterleave( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.deinterleave", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Compute — fma / step / vscale +// ============================================================================= + +/// vector.fma — element-wise fused multiply-add `lhs * rhs + acc`. +pub fn fma( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.fma", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .result_type_inference = true, + .location = location, + }); +} + +/// vector.step — produce `<0, 1, ..., N-1>` of the result type. +pub fn step( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.step", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.vscale — runtime scalable-vector size factor (index-typed). +pub fn vscale(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.vscale", .{ + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// vector.yield — terminator for vector.mask / vector.warp regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Reductions — reduction / multi_reduction / contract +// ============================================================================= + +pub const ReductionOpts = struct { + /// Optional accumulator (scalar of result type). When provided the op is + /// `result = combine(reduce(src), acc)`. + acc: ?*const mlir.Value = null, +}; + +/// vector.reduction — reduce a 1-D vector to a scalar with `kind`. +pub fn reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ReductionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (opts.acc) |a| operands_buf.appendAssumeCapacity(a); + + return mlir.Operation.make(ctx, "vector.reduction", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +/// vector.multi_reduction — reduce a multi-D vector along `reduction_dims`. +pub fn multi_reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + acc: *const mlir.Value, + reduction_dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.multi_reduction", .{ + .operands = .{ .flat = &.{ src, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "reduction_dims", .denseArray(ctx, .i64, reduction_dims)), + }, + .location = location, + }); +} + +pub const ContractOpts = struct { + kind: CombiningKind = .add, +}; + +/// vector.contract — generalized contraction (matmul, dot, batched matmul, +/// reductions). `indexing_maps` is one AffineMapAttr per (lhs, rhs, acc). +/// `iterator_types` is one IteratorType per dim of the iteration space. +pub fn contract( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + indexing_maps: []const *const mlir.Attribute, + iterator_types: []const IteratorType, + result_type: *const mlir.Type, + opts: ContractOpts, + location: *const mlir.Location, +) *mlir.Operation { + var iter_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (iterator_types) |it| iter_attrs.appendAssumeCapacity(it.attribute(ctx)); + + return mlir.Operation.make(ctx, "vector.contract", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "indexing_maps", .array(ctx, indexing_maps)), + .named(ctx, "iterator_types", .array(ctx, iter_attrs.constSlice())), + .named(ctx, "kind", opts.kind.attribute(ctx)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory transfers — load / store / masked / gather / scatter / transfer_* +// ============================================================================= + +pub const LoadStoreOpts = struct { + nontemporal: bool = false, + alignment: ?i64 = null, +}; + +fn loadStoreAttrs(ctx: *mlir.Context, opts: LoadStoreOpts, dst: *stdx.BoundedArray(mlir.NamedAttribute, 2)) void { + if (opts.nontemporal) { + dst.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + if (opts.alignment) |a| { + dst.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } +} + +/// vector.load — strided contiguous load from `base[indices...]` into the +/// result vector. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.store — strided contiguous store of `value` to `base[indices...]`. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.maskedload — like load but lanes whose `mask` bit is 0 are read +/// from `passthru` instead. +pub fn masked_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.maskedload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn masked_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.maskedstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +/// vector.gather — gather scalar elements from `base[indices...]` plus per-lane +/// `index_vec` offsets. +pub fn gather( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, passthru }); + + return mlir.Operation.make(ctx, "vector.gather", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn scatter( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, value }); + + return mlir.Operation.make(ctx, "vector.scatter", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +pub fn expand_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.expandload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn compress_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.compressstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +// ============================================================================= +// Transfer ops — vector.transfer_read / vector.transfer_write +// ============================================================================= + +pub const TransferReadArgs = struct { + /// `affine_map<(d0,...) -> (...)>` attribute. Build with + /// `mlir.parseAffineMapAttribute` or `mlir.Attribute.affineMap(...)`. + permutation_map: *const mlir.Attribute, + /// One bool per result-vector dim. Length must equal vector rank. + in_bounds: []const bool, + /// Optional i1 vector mask. + mask: ?*const mlir.Value = null, +}; + +/// vector.transfer_read — supervector load with permutation map and optional +/// mask. AttrSizedOperandSegments: (base, indices, padding, [mask]). +pub fn transfer_read( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + padding: *const mlir.Value, + args: TransferReadArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendAssumeCapacity(padding); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, @intCast(indices.len), 1, mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); + + return mlir.Operation.make(ctx, "vector.transfer_read", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +pub const TransferWriteArgs = struct { + permutation_map: *const mlir.Attribute, + in_bounds: []const bool, + mask: ?*const mlir.Value = null, + /// When `dest` is a tensor, the op produces a new tensor — pass its type + /// here. For memref destinations leave null. + result_type: ?*const mlir.Type = null, +}; + +/// vector.transfer_write — supervector store with permutation map and optional +/// mask. AttrSizedOperandSegments: (vector, base, indices, [mask]). +pub fn transfer_write( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + args: TransferWriteArgs, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); + + var result_types: stdx.BoundedArray(*const mlir.Type, 1) = .empty; + if (args.result_type) |rt| result_types.appendAssumeCapacity(rt); + + return mlir.Operation.make(ctx, "vector.transfer_write", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +// ============================================================================= +// Masks — constant_mask / create_mask +// ============================================================================= + +/// vector.constant_mask — constant boolean mask. Each entry of +/// `mask_dim_sizes` is the number of leading true lanes along that dim. +pub fn constant_mask( + ctx: *mlir.Context, + mask_dim_sizes: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.constant_mask", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask_dim_sizes", .denseArray(ctx, .i64, mask_dim_sizes)), + }, + .location = location, + }); +} + +/// vector.create_mask — runtime mask from per-dim count operands. `operands` +/// length must equal the rank of `result_type`. +pub fn create_mask( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.create_mask", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} diff --git a/mlir/mlir.h b/mlir/mlir.h index 4ad1523e..b897c82b 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -1,9 +1,15 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include +//===- mlir.h - MLIR C API headers bound by the mlir module -------*- C -*-===// + +#ifndef MLIR_MLIR_H +#define MLIR_MLIR_H + +#include "mlir-c/AffineExpr.h" +#include "mlir-c/AffineMap.h" +#include "mlir-c/BuiltinAttributes.h" +#include "mlir-c/BuiltinTypes.h" +#include "mlir-c/IR.h" +#include "mlir-c/IntegerSet.h" +#include "mlir-c/Pass.h" +#include "mlir-c/Transforms.h" + +#endif // MLIR_MLIR_H diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 6f4bef3b..57e57bd2 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -9,6 +9,57 @@ test { std.testing.refAllDecls(@This()); } +test "variadic operation groups preserve order and dense array segment sizes" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + ctx.setAllowUnregisteredDialects(true); + const loc = Location.unknown(ctx); + const i32_type = Type.int(ctx, .i32); + const i64_type = Type.int(ctx, .i64); + const block = Block.init(&.{ i32_type, i64_type, i32_type }, &.{ loc, loc, loc }); + defer block.deinit(); + const op = Operation.make(ctx, "test.segmented", .{ + .operands = .{ .variadic = &.{ + &.{block.argument(0)}, + &.{}, + &.{ block.argument(1), block.argument(2) }, + } }, + .results = .{ .variadic = &.{ + &.{}, + &.{ i64_type, i32_type }, + &.{i64_type}, + } }, + .location = loc, + }); + defer op.deinit(); + try std.testing.expectEqual(@as(usize, 3), op.numOperands()); + for (0..3) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); + try std.testing.expectEqual(@as(usize, 3), op.numResults()); + try std.testing.expect(op.result(0).type_().eql(i64_type)); + try std.testing.expect(op.result(1).type_().eql(i32_type)); + try std.testing.expect(op.result(2).type_().eql(i64_type)); + try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 0, 2 }))); + try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); +} + +test "unrealized_conversion_cast builds a verified operation" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const loc = Location.unknown(ctx); + const block = Block.init(&.{Type.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + const cast = unrealized_conversion_cast(ctx, &.{block.argument(0)}, &.{Type.index(ctx)}, loc).appendTo(block); + try std.testing.expect(cast.verify()); + try std.testing.expect(cast.result(0).type_().eql(Type.index(ctx))); +} + +test "nested symbol references print their path" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const symbol = Attribute.symbolRef(ctx, "kernels", &.{FlatSymbolRefAttribute.init(ctx, "add")}); + try std.testing.expect(symbol.eql(try Attribute.parse(ctx, "@kernels::@add"))); +} + pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, @@ -51,23 +102,23 @@ pub fn registerPasses(comptime passes: []const u8) void { } fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.is_const; + return @typeInfo(T).pointer.attrs.@"const"; } fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; + return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; } pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; pub const Ptr = if (IsPtrConst) *const T else *T; pub fn ptr(self: anytype) NativeT { return .{ .ptr = @ptrCast(@constCast(self)) }; } - fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { return struct { fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { var ctx: StringCallbackCtx = .{ .writer = writer }; @@ -249,6 +300,14 @@ pub const Context = opaque { return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); } + pub fn setAllowUnregisteredDialects(self: *Context, allow: bool) void { + c.mlirContextSetAllowUnregisteredDialects(self.ptr(), allow); + } + + pub fn allowUnregisteredDialects(self: *const Context) bool { + return c.mlirContextGetAllowUnregisteredDialects(self.ptr()); + } + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ @typeName(Context), @@ -303,7 +362,7 @@ pub const Location = opaque { pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { var buf: [256]u8 = undefined; - return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + return loc.named(ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { buf[buf.len - 3 ..].* = "...".*; break :blk &buf; }); @@ -316,6 +375,77 @@ pub const Location = opaque { pub fn unknown(ctx: *Context) *const Location { return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); } + + pub const Tagged = union(enum) { + unknown, + named: *const Named, + callsite: *const Callsite, + file_line_col: *const FileLineCol, + fused: *const Fused, + }; + + pub fn inspect(location: *const Location) Tagged { + const loc_ptr = location.ptr(); + return if (c.mlirLocationIsAUnknown(loc_ptr)) + .unknown + else if (c.mlirLocationIsAName(loc_ptr)) + .{ .named = @ptrCast(location) } + else if (c.mlirLocationIsACallSite(loc_ptr)) + .{ .callsite = @ptrCast(location) } + else if (c.mlirLocationIsAFileLineColRange(loc_ptr)) + .{ .file_line_col = @ptrCast(location) } + else if (c.mlirLocationIsAFused(loc_ptr)) + .{ .fused = @ptrCast(location) } + else + // Theoritically the C++ API has "OpaqueLoc" but it's not part of C bindings, + // I think in this case it's fine to fall back to .unknown + // https://mlir.llvm.org/docs/Dialects/Builtin/#opaqueloc + .unknown; + } + + pub const Named = opaque { + pub fn name(location: *const Named) []const u8 { + const identifier: *const Identifier = @ptrCast(c.mlirLocationNameGetName(.{ .ptr = location }).ptr); + return identifier.str(); + } + + pub fn childLoc(location: *const Named) *const Location { + return @ptrCast(c.mlirLocationNameGetChildLoc(.{ .ptr = location }).ptr); + } + }; + + pub const Callsite = opaque { + pub fn callee(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCallee(.{ .ptr = location }).ptr); + } + + pub fn caller(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCaller(.{ .ptr = location }).ptr); + } + }; + + pub const FileLineCol = opaque { + pub fn src(location: *const FileLineCol) *const std.builtin.SourceLocation { + const file_id: *const Identifier = @ptrCast(c.mlirLocationFileLineColRangeGetFilename(.{ .ptr = location }).ptr); + return .{ + .file = file_id.str(), + .line = @truncate(c.mlirLocationFileLineColRangeGetStartLine(.{ .ptr = location })), + .column = @truncate(c.mlirLocationFileLineColRangeGetStartColumn(.{ .ptr = location })), + }; + } + }; + + pub const Fused = opaque { + pub fn len(location: *const Fused) u32 { + return @intCast(c.mlirLocationFusedGetNumLocations(.{ .ptr = location })); + } + + pub fn locations(location: *const Fused, allocator: std.mem.Allocator) error{OutOfMemory}![]*const Location { + const result = try allocator.alloc(*const Location, location.len()); + c.mlirLocationFusedGetLocations(.{ .ptr = location }, @ptrCast(result.ptr)); + return result; + } + }; }; pub const Type = opaque { @@ -329,6 +459,56 @@ pub const Type = opaque { pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } + + pub fn isFloat(self: *const Type) bool { + return c.mlirTypeIsAFloat(self.ptr()); + } + + pub fn index(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); + } + + pub fn int(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); + } + + pub fn float(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; + } + + pub fn complex(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); + } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } + + pub fn function(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); + } + + pub fn memRef(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); + } + + pub fn element(self: *const VectorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn vector(shape: []const i64, element_type: *const Type) *const Type { + return @ptrCast(VectorType.init(shape, element_type)); + } + + pub fn unrankedMemRef(element_type: *const Type, memory_space: ?*const Attribute) *const Type { + return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); + } }; pub const IndexType = opaque { @@ -344,10 +524,6 @@ pub const IndexType = opaque { } }; -pub fn indexType(ctx: *Context) *const Type { - return @ptrCast(IndexType.get(ctx)); -} - pub const IntegerTypes = enum { i1, i2, @@ -409,6 +585,10 @@ pub const IntegerType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); + pub fn width(self: *const IntegerType) u32 { + return c.mlirIntegerTypeGetWidth(self.ptr()); + } + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { return exact(ctx, it.bitwidth(), it.signedness()); } @@ -422,10 +602,6 @@ pub const IntegerType = opaque { } }; -pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { - return @ptrCast(IntegerType.get(ctx, it)); -} - pub const FloatTypes = enum { f4e2m1fn, f6e2m3fn, @@ -482,12 +658,6 @@ pub fn FloatType(comptime ft: FloatTypes) type { }; } -pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { - return switch (ft) { - inline else => |v| @ptrCast(FloatType(v).get(ctx)), - }; -} - pub const ComplexTypes = enum { c64, c128, @@ -503,16 +673,12 @@ pub const ComplexType = opaque { pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { - .c64 => floatType(ctx, .f32).ptr(), - .c128 => floatType(ctx, .f64).ptr(), + .c64 => Type.float(ctx, .f32).ptr(), + .c128 => Type.float(ctx, .f64).ptr(), }).ptr); } }; -pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { - return @ptrCast(ComplexType.get(ctx, complex_type)); -} - pub const Attribute = opaque { const M = Methods(Attribute, c.MlirAttribute); @@ -532,6 +698,71 @@ pub const Attribute = opaque { pub fn dialect(self: *const Context) *const Dialect { return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); } + + pub fn string(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); + } + + pub fn int(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); + } + + pub fn intFromType(it: *const IntegerType, value_: i64) *const Attribute { + return @ptrCast(IntegerAttribute.fromType(it, value_)); + } + + pub fn float(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { + return @ptrCast(FloatAttribute.init(ctx, ft, value_)); + } + pub fn boolean(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); + } + + pub fn typeAttr(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); + } + + pub fn dict(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); + } + + pub fn denseElements(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); + } + + pub fn denseArray(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); + } + pub fn array(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); + } + + pub fn intArray(ctx: *Context, comptime T: type, values: []const T) *const Attribute { + const it: IntegerTypes = comptime @field(IntegerTypes, @typeName(T)); + var buf: stdx.BoundedArray(*const Attribute, ShapedType.MAX_RANK) = .empty; + for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); + return .array(ctx, buf.constSlice()); + } + + pub fn flatSymbolRef(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); + } + + pub fn symbolRef(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const Attribute { + return @ptrCast(SymbolRefAttribute.init(ctx, root, nested)); + } + + pub fn unit(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); + } + + pub fn affineMap(map: *const AffineMap) *const Attribute { + return @ptrCast(AffineMapAttribute.init(map)); + } + + pub fn integerSet(set: *const IntegerSet) *const Attribute { + return @ptrCast(IntegerSetAttribute.init(set)); + } }; pub const StringAttribute = opaque { @@ -551,10 +782,6 @@ pub const StringAttribute = opaque { } }; -pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { - return @ptrCast(StringAttribute.init(ctx, str)); -} - pub const IntegerAttribute = opaque { const M = Methods(IntegerAttribute, c.MlirAttribute); @@ -564,10 +791,11 @@ pub const IntegerAttribute = opaque { pub const format = M.format(c.mlirAttributePrint); pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { - return @ptrCast(c.mlirIntegerAttrGet( - IntegerType.get(ctx, it).ptr(), - @intCast(value_), - ).ptr); + return .fromType(.get(ctx, it), @intCast(value_)); + } + + pub fn fromType(it: *const IntegerType, value_: i64) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet(it.ptr(), value_).ptr); } pub fn value(self: *const IntegerAttribute, comptime T: type) T { @@ -579,10 +807,6 @@ pub const IntegerAttribute = opaque { } }; -pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { - return @ptrCast(IntegerAttribute.init(ctx, it, value_)); -} - pub const FloatAttribute = opaque { const M = Methods(FloatAttribute, c.MlirAttribute); @@ -600,10 +824,6 @@ pub const FloatAttribute = opaque { } }; -pub fn floatAttribute(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { - return @ptrCast(FloatAttribute.init(ctx, ft, value_)); -} - pub const BoolAttribute = opaque { const M = Methods(BoolAttribute, c.MlirAttribute); @@ -621,10 +841,6 @@ pub const BoolAttribute = opaque { } }; -pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { - return @ptrCast(BoolAttribute.init(ctx, value)); -} - pub const TypeAttribute = opaque { const M = Methods(TypeAttribute, c.MlirAttribute); @@ -642,10 +858,6 @@ pub const TypeAttribute = opaque { } }; -pub fn typeAttribute(type_: *const Type) *const Attribute { - return @ptrCast(TypeAttribute.init(type_)); -} - pub const Dialect = struct { const M = Methods(Dialect, c.MlirDialect); @@ -797,10 +1009,24 @@ pub const Block = opaque { } } + pub fn firstOperation(self: *const Block) ?*Operation { + const op = c.mlirBlockGetFirstOperation(self.ptr()); + if (op.ptr == null) return null; + return @ptrCast(op.ptr); + } + + pub fn insertOwnedOperationBefore(self: *Block, reference: *Operation, op: *Operation) void { + c.mlirBlockInsertOwnedOperationBefore(self.ptr(), reference.ptr(), op.ptr()); + } + pub fn parentOperation(self: *const Block) ?*Operation { return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); } + pub fn parentRegion(self: *const Block) *Region { + return @ptrCast(c.mlirBlockGetParentRegion(self.ptr()).ptr); + } + pub fn detach(self: *Block) void { c.mlirBlockDetach(self.ptr()); } @@ -881,10 +1107,6 @@ pub const DictionaryAttribute = opaque { } }; -pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { - return @ptrCast(DictionaryAttribute.init(ctx, attributes)); -} - pub const Value = opaque { const M = Methods(Value, c.MlirValue); @@ -1026,7 +1248,7 @@ pub const Operation = opaque { } pub fn name(self: *const Operation) []const u8 { - return string(c.mlirOperationGetName(self.ptr())); + return string(c.mlirIdentifierStr(c.mlirOperationGetName(self.ptr()))); } pub fn clone(self: *const Operation) *Operation { @@ -1042,6 +1264,14 @@ pub const Operation = opaque { return self; } + pub fn numRegions(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumRegions(self.ptr())); + } + + pub fn region(self: *const Operation, index: usize) *Region { + return @ptrCast(c.mlirOperationGetRegion(self.ptr(), @intCast(index)).ptr); + } + pub fn parent(self: *const Operation) ?*Operation { return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); } @@ -1082,6 +1312,16 @@ pub const Operation = opaque { c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); } + /// Sets `attribute` under its name, as a dialect's attribute helper names it. + pub fn setAttribute(self: *Operation, attribute: NamedAttribute) void { + self.setAttributeByName(attribute.name().str(), attribute.attribute()); + } + + pub fn attributeByName(self: *const Operation, name_: []const u8) ?*const Attribute { + const attr = c.mlirOperationGetAttributeByName(self.ptr(), stringRef(name_)); + return if (attr.ptr == null) null else @ptrCast(attr.ptr); + } + pub const PrintFlags = struct { elide_large_elements_attrs: ?usize = null, debug_info: bool = false, @@ -1089,6 +1329,7 @@ pub const Operation = opaque { print_generic_op_form: bool = false, use_local_scope: bool = false, assume_verified: bool = false, + print_name_loc_as_prefix: bool = false, const Ctx = struct { self: *const Operation, @@ -1114,6 +1355,9 @@ pub const Operation = opaque { if (self.assume_verified) { c.mlirOpPrintingFlagsAssumeVerified(flags); } + if (self.print_name_loc_as_prefix) { + c.mlirOpPrintingFlagsPrintNameLocAsPrefix(flags); + } return flags; } }; @@ -1154,7 +1398,7 @@ pub const Operation = opaque { attributes: ?[]const NamedAttribute = null, blocks: ?[]const *Block = null, verify: bool = true, - location: ?*const Location = null, + location: ?*const Location, }; pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { @@ -1165,26 +1409,26 @@ pub const Operation = opaque { if (args.operands) |operands| switch (operands) { .flat => |v| state.addOperands(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; for (segments) |segment_operands| { state.addOperands(segment_operands); sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), }); }, }; if (args.results) |results| switch (results) { .flat => |v| state.addResults(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; for (segments) |segment_results| { state.addResults(segment_results); sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), + .named(ctx, "resultSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), }); }, }; @@ -1192,11 +1436,11 @@ pub const Operation = opaque { state.addAttributes(attrs); } if (args.blocks) |blocks| { - const region = Region.init(); for (blocks) |block_| { - region.appendOwnedBlock(block_); + const body_region = Region.init(); + body_region.appendOwnedBlock(block_); + state.addOwnedRegions(&.{body_region}); } - state.addOwnedRegions(&.{region}); } const new_op = try Operation.init(&state); errdefer new_op.deinit(); @@ -1246,11 +1490,11 @@ pub const Operation = opaque { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @intFromEnum(order), + @backingInt(order), ); } @@ -1304,6 +1548,16 @@ pub const Operation = opaque { } }; +/// builtin.unrealized_conversion_cast — `inputs` as values of `outputs_types`, +/// a cast between type systems that a later conversion resolves. +pub fn unrealized_conversion_cast(ctx: *Context, inputs: []const *const Value, outputs_types: []const *const Type, location: *const Location) *Operation { + return Operation.make(ctx, "builtin.unrealized_conversion_cast", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = outputs_types }, + .location = location, + }); +} + pub const Region = opaque { const M = Methods(Region, c.MlirRegion); @@ -1369,6 +1623,14 @@ pub const ShapedType = opaque { pub fn dimension(self: *const ShapedType, dim: usize) i64 { return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } }; pub const RankedTensorType = opaque { @@ -1379,23 +1641,24 @@ pub const RankedTensorType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { + pub fn get(dimensions: []const i64, elem_type: *const Type, encoding: ?*Attribute) *const RankedTensorType { return @ptrCast(c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions), - elemType.ptr(), + elem_type.ptr(), if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), ).ptr); } pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } return get( dims.constSlice(), other.elementType(), + null, ); } @@ -1416,14 +1679,6 @@ pub const RankedTensorType = opaque { } }; -pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); -} - -pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); -} - pub const DenseElementsAttribute = opaque { const M = Methods(DenseElementsAttribute, c.MlirAttribute); @@ -1442,10 +1697,6 @@ pub const DenseElementsAttribute = opaque { } }; -pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { - return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); -} - pub const DenseArrayTypes = enum { bool, i8, @@ -1505,10 +1756,6 @@ pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { }; } -pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { - return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); -} - pub const ArrayAttribute = opaque { const M = Methods(ArrayAttribute, c.MlirAttribute); @@ -1530,10 +1777,6 @@ pub const ArrayAttribute = opaque { } }; -pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { - return @ptrCast(ArrayAttribute.init(ctx, attrs)); -} - pub const FunctionType = opaque { const M = Methods(FunctionType, c.MlirType); pub const isAFn = c.mlirTypeIsAFunction; @@ -1553,10 +1796,6 @@ pub const FunctionType = opaque { } }; -pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { - return @ptrCast(FunctionType.init(ctx, args, results)); -} - pub const FlatSymbolRefAttribute = opaque { const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); @@ -1574,9 +1813,19 @@ pub const FlatSymbolRefAttribute = opaque { } }; -pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { - return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); -} +/// `@root::@nested...`. +pub const SymbolRefAttribute = opaque { + const M = Methods(SymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsASymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const SymbolRefAttribute { + return @ptrCast(c.mlirSymbolRefAttrGet(ctx.ptr(), stringRef(root), @intCast(nested.len), @ptrCast(nested)).ptr); + } +}; pub const UnitAttribute = opaque { const M = Methods(UnitAttribute, c.MlirAttribute); @@ -1591,30 +1840,27 @@ pub const UnitAttribute = opaque { } }; -pub fn unitAttribute(ctx: *Context) *const Attribute { - return @ptrCast(UnitAttribute.get(ctx)); -} - pub const MemRefType = opaque { const M = Methods(MemRefType, c.MlirType); - pub const isAFn = c.mlirTypeIsAInteger; + pub const isAFn = c.mlirTypeIsAMemRef; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + pub fn init(element_type: *const Type, shape: []const i64, layout_attr: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { return @ptrCast(c.mlirMemRefTypeGet( element_type.ptr(), @intCast(shape.len), @ptrCast(shape), - if (layout) |l| l.ptr() else .{}, - if (memory_space) |m| m.ptr() else .{}, + if (layout_attr) |l| l.ptr() else .{ .ptr = null }, + if (memory_space) |m| m.ptr() else .{ .ptr = null }, ).ptr); } pub fn fromShaped(other: *const ShapedType) *const MemRefType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } @@ -1626,11 +1872,244 @@ pub const MemRefType = opaque { ); } + pub fn elementType(self: *const MemRefType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const MemRefType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const MemRefType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn layout(self: *const MemRefType) *const Attribute { + return @ptrCast(c.mlirMemRefTypeGetLayout(self.ptr()).ptr); + } + + pub fn memorySpace(self: *const MemRefType) ?*const Attribute { + const a = c.mlirMemRefTypeGetMemorySpace(self.ptr()); + return if (a.ptr == null) null else @ptrCast(a.ptr); + } + pub fn shaped(self: *const MemRefType) *const ShapedType { return @ptrCast(self); } }; -pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { - return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); -} +pub const VectorType = opaque { + const M = Methods(VectorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAVector; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// Plain (non-scalable) vector type. + pub fn init(shape: []const i64, element_type: *const Type) *const VectorType { + return @ptrCast(c.mlirVectorTypeGet( + @intCast(shape.len), + @ptrCast(shape), + element_type.ptr(), + ).ptr); + } + + /// Vector type with explicit per-dim scalability flags. `scalable.len` + /// must equal `shape.len`. + pub fn initScalable( + shape: []const i64, + scalable: []const bool, + element_type: *const Type, + ) *const VectorType { + std.debug.assert(shape.len == scalable.len); + return @ptrCast(c.mlirVectorTypeGetScalable( + @intCast(shape.len), + @ptrCast(shape), + @ptrCast(scalable), + element_type.ptr(), + ).ptr); + } + + pub fn rank(self: *const VectorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const VectorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const VectorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub const UnrankedMemRefType = opaque { + const M = Methods(UnrankedMemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAUnrankedMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn init(element_type: *const Type, memory_space: ?*const Attribute) *const UnrankedMemRefType { + return @ptrCast(c.mlirUnrankedMemRefTypeGet( + element_type.ptr(), + if (memory_space) |m| m.ptr() else .{ .ptr = null }, + ).ptr); + } +}; + +pub const AffineExpr = opaque { + const M = Methods(AffineExpr, c.MlirAffineExpr); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineExprEqual); + pub const format = M.format(c.mlirAffineExprPrint); + + pub fn dim(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineDimExprGet(ctx.ptr(), @intCast(position)).ptr); + } + + pub fn symbol(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineSymbolExprGet(ctx.ptr(), @intCast(position)).ptr); + } + + pub fn constant(ctx: *Context, value: i64) *const AffineExpr { + return @ptrCast(c.mlirAffineConstantExprGet(ctx.ptr(), value).ptr); + } + + pub fn add(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineAddExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mul(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineMulExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn floorDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineFloorDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn ceilDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineCeilDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mod(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineModExprGet(self.ptr(), other.ptr()).ptr); + } +}; + +pub const AffineMap = opaque { + const M = Methods(AffineMap, c.MlirAffineMap); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineMapEqual); + pub const format = M.format(c.mlirAffineMapPrint); + + /// Empty `() -> ()` map. + pub fn empty(ctx: *Context) *const AffineMap { + return @ptrCast(c.mlirAffineMapEmptyGet(ctx.ptr()).ptr); + } + + /// Zero-result map: `(d0, d1, ...) -> ()`. + pub fn zeroResult(ctx: *Context, dim_count: usize, symbol_count: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapZeroResultGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + ).ptr); + } + + /// Identity map: `(d0, d1, ..., dN-1) -> (d0, d1, ..., dN-1)`. + pub fn multiDimIdentity(ctx: *Context, num_dims: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapMultiDimIdentityGet(ctx.ptr(), @intCast(num_dims)).ptr); + } + + /// Build a map from explicit results. + pub fn get( + ctx: *Context, + dim_count: usize, + symbol_count: usize, + results: []const *const AffineExpr, + ) *const AffineMap { + return @ptrCast(c.mlirAffineMapGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + @intCast(results.len), + @ptrCast(@constCast(results.ptr)), + ).ptr); + } + + pub fn numDims(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumDims(self.ptr())); + } + + pub fn numSymbols(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumSymbols(self.ptr())); + } + + pub fn numResults(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumResults(self.ptr())); + } +}; + +/// AffineMapAttr — `affine_map<(d0,...) -> (...)>` wrapped as an attribute. +pub const AffineMapAttribute = opaque { + const M = Methods(AffineMapAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAAffineMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(map: *const AffineMap) *const AffineMapAttribute { + return @ptrCast(c.mlirAffineMapAttrGet(map.ptr()).ptr); + } + + pub fn value(self: *const AffineMapAttribute) *const AffineMap { + return @ptrCast(c.mlirAffineMapAttrGetValue(self.ptr()).ptr); + } +}; + +pub const IntegerSet = opaque { + const M = Methods(IntegerSet, c.MlirIntegerSet); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIntegerSetEqual); + pub const format = M.format(c.mlirIntegerSetPrint); + + pub fn get( + ctx: *Context, + num_dims: usize, + num_symbols: usize, + constraints: []const *const AffineExpr, + eqs: []const bool, + ) *const IntegerSet { + std.debug.assert(constraints.len == eqs.len); + return @ptrCast(c.mlirIntegerSetGet( + ctx.ptr(), + @intCast(num_dims), + @intCast(num_symbols), + @intCast(constraints.len), + @ptrCast(constraints), + @ptrCast(eqs), + ).ptr); + } +}; + +pub const IntegerSetAttribute = opaque { + const M = Methods(IntegerSetAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAIntegerSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(set: *const IntegerSet) *const IntegerSetAttribute { + return @ptrCast(c.mlirIntegerSetAttrGet(set.ptr()).ptr); + } +}; diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index e9f365d1..5370afdc 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -4,6 +4,7 @@ zig_library( name = "pjrt", srcs = ["ffi.zig"], main = "pjrt.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "@xla//xla/ffi/api:c_api", @@ -19,6 +20,5 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_raw_buffer_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", - "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index 83fad4d2..cebec841 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -11,7 +11,7 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); comptime { - if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); + if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); } /// The signature of a generic custom call. @@ -27,7 +27,7 @@ pub const ApiVersion = extern struct { minor_version: i32, }; -pub const ExtensionType = enum(c.XLA_FFI_Extension_Type) { +pub const ExtensionType = enum(c.XLA_FFI_InternalExtension_Type) { metadata = c.XLA_FFI_Extension_Metadata, }; @@ -152,17 +152,17 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { }; pub const ExecutionContext = opaque { - pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; + pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_InvokeContext).to; pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { - var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ - .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), + var ret: c.XLA_FFI_InvokeContext_Get_Args = .{ + .struct_size = pjrt.meta.structSize(c.XLA_FFI_InvokeContext_Get_Args), .extension_start = api.inner().extension_start, .ctx = @ptrCast(@constCast(self)), .type_id = @constCast(&type_id), - .data = undefined, // set by XLA_FFI_ExecutionContext_Get. + .data = undefined, // set by XLA_FFI_InvokeContext_Get. }; - const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); + const maybe_err = api.inner().XLA_FFI_InvokeContext_Get.?(&ret); if (maybe_err) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -252,11 +252,12 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3b11fnuz = c.XLA_FFI_DataType_F8E4M3B11FNUZ, f8e5m2fnuz = c.XLA_FFI_DataType_F8E5M2FNUZ, f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, + f4e2m1 = c.XLA_FFI_DataType_F4E2M1FN, }; pub const Buffer = extern struct { struct_size: usize, - extension_start: ?*c.XLA_FFI_Extension_Base, + extension_start: ?*c.XLA_FFI_InternalExtension, dtype: DataType, data: [*]u8, rank: u64, @@ -273,7 +274,7 @@ pub const Buffer = extern struct { pub const Args = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_Extension_Base, + extension_start: ?*const c.XLA_FFI_InternalExtension, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -289,7 +290,7 @@ pub const Args = extern struct { pub const Rets = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_Extension_Base, + extension_start: ?*const c.XLA_FFI_InternalExtension, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -449,7 +450,7 @@ pub const Error = opaque { pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, - .errc = @intFromEnum(error_code), + .errc = @backingInt(error_code), }; return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } @@ -460,10 +461,10 @@ pub const Error = opaque { } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetMessage_Args) = .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetDetails_Args) = .{ .@"error" = err.inner(), }; - api.inner().XLA_FFI_Error_GetMessage.?(@ptrCast(&ret)); + api.inner().XLA_FFI_Error_GetDetails.?(@ptrCast(&ret)); return std.mem.span(ret.message); } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 073691e5..3715361f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -5,7 +5,8 @@ const c = @import("c"); pub const ffi = @import("ffi.zig"); -const log = std.log.scoped(.pjrt); +const log_scope = .pjrt; +const log = std.log.scoped(log_scope); test { std.testing.refAllDecls(@This()); @@ -31,18 +32,18 @@ pub const meta = struct { } pub fn Struct(comptime T: type) type { - const fields = std.meta.fields(T); - var names: [fields.len][]const u8 = undefined; - var types: [fields.len]type = undefined; - var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { - name.* = field.name; - type_.* = field.type; + const info = @typeInfo(T).@"struct"; + var names: [info.field_names.len][]const u8 = undefined; + var types: [info.field_names.len]type = undefined; + var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { + name.* = field_name; + type_.* = Field; attr.* = .{ - .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) &structSize(T) else - &std.mem.zeroes(field.type)), + &std.mem.zeroes(Field)), }; } return @Struct( @@ -74,6 +75,15 @@ pub const ApiError = error{ Unauthenticated, }; +inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8, comptime log_level: std.log.Level) ApiError { + defer pjrt_error.deinit(api); + const err_code = pjrt_error.getCode(api).toApiError(); + if (comptime std.log.logEnabled(log_level, log_scope)) { + std.options.logFn(log_level, log_scope, "[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + } + return err_code; +} + fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { @@ -99,7 +109,6 @@ pub const Api = struct { pub fn loadFrom(library: [:0]const u8) !*const Api { const basename = std.Io.Dir.path.basename(library); log.info("Loading: {s}...", .{basename}); - defer log.info("Loaded: {s}", .{basename}); var lib: std.DynLib = switch (builtin.os.tag) { .linux, .macos => blk: { @@ -112,28 +121,33 @@ pub const Api = struct { break :blk .{ .inner = .{ .handle = std.c.dlopen(library, rtld) orelse { - log.err("Unable to dlopen plugin: {s}", .{library}); + log.err("Unable to dlopen plugin {s}\n{s}", .{ library, std.c.dlerror() orelse "" }); return error.FileNotFound; }, }, }; }, else => std.DynLib.open(library) catch |err| { - log.err("Unable to dlopen plugin: {s}", .{library}); + log.err("Unable to dlopen plugin {s}: {}", .{ library, err }); return err; }, }; - return fromDynLib(&lib); + const api = fromDynLib(&lib) catch |err| { + log.err("Unable to load PJRT API from plugin {s}: {}", .{ library, err }); + return err; + }; + log.info("Loaded: {s}", .{basename}); + return api; } pub fn fromDynLib(lib: *std.DynLib) !*const Api { const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { - std.debug.panic("Unable to find GetPjrtApi symbol in library", .{}); + return error.MissingGetPjrtApi; }; const api = DynGetPjrtApi(); - _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; + _ = try api.call(.PJRT_Plugin_Initialize, .{}); return api; } @@ -142,7 +156,7 @@ pub const Api = struct { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); return arg_array_type_info.pointer.child; } @@ -154,25 +168,33 @@ pub const Api = struct { }; } - inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { + inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ?*Error { if (@offsetOf(c.PJRT_Api, @tagName(method)) > self.inner.struct_size) { std.debug.panic("PJRT Api method {s} not available in this plugin", .{@tagName(method)}); } const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(arg); if (@TypeOf(result) == void) { - return; + return null; } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); - log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(self) }); - return pjrt_error.getCode(self).toApiError(); + return pjrt_error; } + return null; } inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { var ret = arg; - try innerCall(self, method, @ptrCast(&ret)); + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .warn); + return ret; + } + + inline fn callSilent(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { + var ret = arg; + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .debug); return ret; } @@ -180,12 +202,10 @@ pub const Api = struct { pub const Type = enum(c.PJRT_Extension_Type) { custom_call = c.PJRT_Extension_Type_Gpu_Custom_Call, profiler = c.PJRT_Extension_Type_Profiler, - custom_partitioner = c.PJRT_Extension_Type_Custom_Partitioner, stream = c.PJRT_Extension_Type_Stream, layouts = c.PJRT_Extension_Type_Layouts, ffi = c.PJRT_Extension_Type_FFI, memory_descriptions = c.PJRT_Extension_Type_MemoryDescriptions, - triton = c.PJRT_Extension_Type_Triton, raw_buffer = c.PJRT_Extension_Type_RawBuffer, phase_compile = c.PJRT_Extension_Type_PhaseCompile, unknown = c.PJRT_Extension_Type_Unknown, @@ -194,12 +214,10 @@ pub const Api = struct { pub const Extension = union(Type) { custom_call: *const c.PJRT_Gpu_Custom_Call, profiler: *const c.PJRT_Profiler_Extension, - custom_partitioner: *const c.PJRT_Custom_Partitioner_Extension, stream: *const c.PJRT_Stream_Extension, layouts: *const c.PJRT_Layouts_Extension, ffi: *const c.PJRT_FFI_Extension, memory_descriptions: *const c.PJRT_MemoryDescriptions_Extension, - triton: *const c.PJRT_Triton_Extension, raw_buffer: *const c.PJRT_RawBuffer_Extension, phase_compile: *const c.PJRT_PhaseCompile_Extension, unknown: *const c.PJRT_Extension_Base, @@ -249,15 +267,15 @@ pub const Api = struct { pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: ?[:0]const u8 = null; + var str: ?[]const u8 = null; }; if (state.str) |str| { return str; } - if (self.getPluginAttribute("stablehlo_current_version")) |nv| { + if (self.pluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { - state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; }, else => unreachable, } @@ -277,8 +295,16 @@ pub const Api = struct { return null; } - fn getPluginAttribute(api: *const Api, key: []const u8) ?NamedValue { - const attributes = api.getPluginAttributes(); + pub fn profiler(self: *const Api, options_pb: []const u8) ApiError!?Profiler { + if (self.extension(.profiler)) |ext| { + return try Profiler.init(self, ext.profiler, options_pb); + } + + return null; + } + + pub fn pluginAttribute(api: *const Api, key: []const u8) ?NamedValue { + const attributes = api.pluginAttributes(); for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), key)) { return attr; @@ -287,7 +313,7 @@ pub const Api = struct { return null; } - fn getPluginAttributes(api: *const Api) []const NamedValue { + pub fn pluginAttributes(api: *const Api) []const NamedValue { const ret = api.call(.PJRT_Plugin_Attributes, .{ .extension_start = null, }) catch unreachable; @@ -339,22 +365,24 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { }; pub const Error = opaque { + const inner = InnerMixin(c.PJRT_Error).inner; + pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; } pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; - return @enumFromInt(ret.code); + return @fromBackingInt(@intCast(ret.code)); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -374,7 +402,7 @@ pub const ShapeSpec = extern struct { .inner = .{ .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, - .element_type = @intFromEnum(bt), + .element_type = @backingInt(bt), }, }; } @@ -384,7 +412,7 @@ pub const ShapeSpec = extern struct { } pub fn bufferType(self: ShapeSpec) BufferType { - return @enumFromInt(self.inner.element_type); + return @fromBackingInt(@intCast(self.inner.element_type)); } }; @@ -466,6 +494,7 @@ pub const Client = opaque { buffer_type: BufferType, dims: []const i64, byte_strides: ?[]const i64, + layout: MemoryLayout, host_buffer_semantics: HostBufferSemantics, dst: union(enum) { device: *const Device, @@ -477,14 +506,15 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @constCast(args.data), - .type = @intFromEnum(args.buffer_type), + .type = @backingInt(args.buffer_type), .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, - .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), + .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), + .host_buffer_semantics = @backingInt(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); return .{ @@ -503,11 +533,14 @@ pub const Client = opaque { } pub const CreateViewOfDeviceBufferArgs = struct { - data: *anyopaque, + device_buffer_ptr: *anyopaque, dims: []const i64, element_type: BufferType, layout: MemoryLayout, device: ?*const Device = null, + /// The memory holding `device_buffer_ptr`. Takes precedence over `device`, + /// which designates its default memory. + memory: ?*const Memory = null, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} }.call, @@ -519,15 +552,16 @@ pub const Client = opaque { const layout = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), - .device_buffer_ptr = @constCast(args.data), + .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @intFromEnum(args.element_type), + .element_type = @backingInt(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = @bitCast(@intFromPtr(args.stream)), + .memory = if (args.memory) |memory| memory.inner() else null, }); return @ptrCast(ret.buffer.?); } @@ -570,11 +604,68 @@ pub const Client = opaque { .num_shape_specs = args.shape_specs.len, .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts)) else null, .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, - .memory = @ptrCast(@constCast(args.memory)), + .memory = args.memory.inner(), }); return @ptrCast(ret.transfer_manager.?); } + pub fn defaultMemoryLayout( + self: *const Client, + api: *const Api, + element_type: BufferType, + dims: []const i64, + ) !DefaultMemoryLayout { + const ext = if (api.extension(.layouts)) |e| e.layouts else return error.LayoutsExtensionUnavailable; + + var get_args: c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args = .{ + .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, + .extension_start = null, + .client = self.inner(), + .type = @backingInt(element_type), + .dims = dims.ptr, + .num_dims = dims.len, + .layout = null, + }; + if (ext.PJRT_Layouts_PJRT_Client_GetDefaultLayout.?(&get_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer { + var destroy_args: c.PJRT_Layouts_MemoryLayout_Destroy_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Destroy_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + }; + if (ext.PJRT_Layouts_MemoryLayout_Destroy.?(&destroy_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + log.err("[PJRT_Layouts_MemoryLayout_Destroy] {s}", .{pjrt_error.getMessage(api)}); + } + } + + var serialize_args: c.PJRT_Layouts_MemoryLayout_Serialize_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Serialize_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + .serialized_bytes = null, + .serialized_bytes_size = 0, + .serialized_layout = null, + .serialized_layout_deleter = null, + }; + if (ext.PJRT_Layouts_MemoryLayout_Serialize.?(&serialize_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer if (serialize_args.serialized_layout_deleter) |deleter| { + deleter(serialize_args.serialized_layout); + }; + + const serialized = serialize_args.serialized_bytes[0..serialize_args.serialized_bytes_size]; + return try DefaultMemoryLayout.parseSerialized(serialized); + } + pub const CreateUninitializedBufferArgs = struct { dims: []const i64, element_type: BufferType, @@ -591,10 +682,10 @@ pub const Client = opaque { .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, - .shape_element_type = @intFromEnum(args.element_type), + .shape_element_type = @backingInt(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); return @ptrCast(ret.buffer.?); } @@ -603,6 +694,40 @@ pub const Client = opaque { pub const Device = opaque { const inner = InnerMixin(c.PJRT_Device).inner; + pub const Attributes = struct { + values: []const NamedValue, + state: *c.PJRT_Device_Attributes, + deleter: *const fn (?*c.PJRT_Device_Attributes) callconv(.c) void, + + pub fn deinit(self: Attributes) void { + self.deleter(self.state); + } + + /// String and list values remain valid until this owner is deinitialized. + pub fn get(self: Attributes, name: []const u8) ?NamedValue.Value { + for (self.values) |attribute| { + if (std.mem.eql(u8, attribute.name(), name)) return attribute.value(); + } + return null; + } + }; + + pub fn attributes(self: *const Device, api: *const Api) ApiError!Attributes { + const ret = try api.call(.PJRT_Device_GetAttributes, .{ .device = self.inner() }); + + const deleter = ret.attributes_deleter.?; + errdefer deleter(ret.device_attributes); + + const state = ret.device_attributes.?; + std.debug.assert((ret.attributes == null) == (ret.num_attributes == 0)); + + return .{ + .values = if (ret.attributes == null) &.{} else @ptrCast(ret.attributes[0..ret.num_attributes]), + .state = state, + .deleter = deleter, + }; + } + pub fn getDescription(self: *const Device, api: *const Api) *const DeviceDescription { const ret = api.call(.PJRT_Device_GetDescription, .{ .device = self.inner(), @@ -692,10 +817,11 @@ pub const Device = opaque { }; pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { - const ret = try api.call(.PJRT_Device_MemoryStats, .{ + // callSilent: If this API is not available no point in printing it at every call + const ret = try api.callSilent(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(@bitCast(ret)); + return .fromCStruct(ret); } }; @@ -741,6 +867,9 @@ pub const DeviceDescription = opaque { const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ .device_description = self.inner(), }) catch unreachable; + + if (ret.attributes == null) return &.{}; + return @ptrCast(ret.attributes[0..ret.num_attributes]); } @@ -991,6 +1120,128 @@ pub const MemoryLayout = union(MemoryLayoutType) { }, }; } + + fn fromCStruct(self: c.PJRT_Buffer_MemoryLayout) MemoryLayout { + return switch (self.type) { + c.PJRT_Buffer_MemoryLayout_Type_Tiled => b: { + const tiled = self.unnamed_0.tiled; + + const minor_to_major: []const i64 = if (tiled.minor_to_major == null) &.{} else tiled.minor_to_major[0..tiled.minor_to_major_size]; + const tile_dims_sizes: []const usize = if (tiled.tile_dim_sizes == null) &.{} else tiled.tile_dim_sizes[0..tiled.num_tiles]; + + const tile_dims: []const i64 = if (tiled.tile_dims == null) &.{} else td: { + var tile_dim_len: usize = 0; + for (tile_dims_sizes) |size| tile_dim_len += size; + break :td tiled.tile_dims[0..tile_dim_len]; + }; + + break :b .{ + .tiled = .{ + .minor_to_major = minor_to_major, + .tile_dims = tile_dims, + .tile_dims_sizes = tile_dims_sizes, + }, + }; + }, + c.PJRT_Buffer_MemoryLayout_Type_Strides => .{ + .strides = .{ .byte_strides = self.unnamed_0.strides.byte_strides[0..self.unnamed_0.strides.num_byte_strides] }, + }, + else => unreachable, + }; + } +}; + +pub const DefaultMemoryLayout = struct { + pub const MAX_MINOR_TO_MAJOR: usize = 16; + pub const MAX_TILE_DIMS: usize = 128; + pub const MAX_NUM_TILES: usize = 16; + + minor_to_major_storage: [MAX_MINOR_TO_MAJOR]i64 = undefined, + minor_to_major_len: usize = 0, + + tile_dims_storage: [MAX_TILE_DIMS]i64 = undefined, + tile_dims_len: usize = 0, + + tile_dims_sizes_storage: [MAX_NUM_TILES]usize = undefined, + tile_dims_sizes_len: usize = 0, + + pub fn toMemoryLayout(self: *const DefaultMemoryLayout) MemoryLayout { + return .{ + .tiled = .{ + .minor_to_major = self.minor_to_major_storage[0..self.minor_to_major_len], + .tile_dims = self.tile_dims_storage[0..self.tile_dims_len], + .tile_dims_sizes = self.tile_dims_sizes_storage[0..self.tile_dims_sizes_len], + }, + }; + } + + pub fn parseSerialized(serialized: []const u8) !DefaultMemoryLayout { + var parsed: DefaultMemoryLayout = .{}; + try parseSerializedInto(serialized, &parsed); + return parsed; + } + + fn parseSerializedInto(serialized: []const u8, out: *DefaultMemoryLayout) !void { + out.* = .{}; + const raw = std.mem.trim(u8, serialized, " \t\n\r\x00"); + + if (raw.len < 2 or raw[0] != '{' or raw[raw.len - 1] != '}') { + return error.InvalidLayoutString; + } + + const body = raw[1 .. raw.len - 1]; + const first_colon = std.mem.indexOfScalar(u8, body, ':'); + + const minor_to_major_str = std.mem.trim(u8, if (first_colon) |index| body[0..index] else body, " \t\n\r"); + + if (minor_to_major_str.len > 0) { + var mtm_iter = std.mem.splitScalar(u8, minor_to_major_str, ','); + while (mtm_iter.next()) |part_raw| { + if (out.minor_to_major_len >= MAX_MINOR_TO_MAJOR) return error.LayoutTooLarge; + out.minor_to_major_storage[out.minor_to_major_len] = try parseI64Token(part_raw); + out.minor_to_major_len += 1; + } + } + + if (first_colon) |index| { + const attrs = body[index + 1 ..]; + var cursor: usize = 0; + while (cursor < attrs.len) { + const t_pos = std.mem.indexOfScalarPos(u8, attrs, cursor, 'T') orelse break; + cursor = t_pos + 1; + while (cursor < attrs.len and std.ascii.isWhitespace(attrs[cursor])) : (cursor += 1) {} + if (cursor >= attrs.len or attrs[cursor] != '(') continue; + + while (cursor < attrs.len and attrs[cursor] == '(') { + const tuple_start = cursor + 1; + const tuple_end = std.mem.indexOfScalarPos(u8, attrs, tuple_start, ')') orelse return error.InvalidLayoutString; + const tuple = attrs[tuple_start..tuple_end]; + + var dims_in_tile: usize = 0; + var tuple_iter = std.mem.splitScalar(u8, tuple, ','); + while (tuple_iter.next()) |dim_raw| { + if (out.tile_dims_len >= MAX_TILE_DIMS) return error.LayoutTooLarge; + out.tile_dims_storage[out.tile_dims_len] = try parseI64Token(dim_raw); + out.tile_dims_len += 1; + dims_in_tile += 1; + } + if (dims_in_tile == 0) return error.InvalidLayoutString; + + if (out.tile_dims_sizes_len >= MAX_NUM_TILES) return error.LayoutTooLarge; + out.tile_dims_sizes_storage[out.tile_dims_sizes_len] = dims_in_tile; + out.tile_dims_sizes_len += 1; + + cursor = tuple_end + 1; + } + } + } + } + + fn parseI64Token(token_raw: []const u8) !i64 { + const token = std.mem.trim(u8, token_raw, " \t\n\r"); + if (token.len == 0) return error.InvalidLayoutString; + return std.fmt.parseInt(i64, token, 10); + } }; pub const HostBufferSemantics = enum(c.PJRT_HostBufferSemantics) { @@ -1049,7 +1300,7 @@ pub const Buffer = opaque { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @enumFromInt(ret.type); + return @fromBackingInt(@intCast(ret.type)); } pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -1095,7 +1346,10 @@ pub const Buffer = opaque { const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); - return ret.device_memory_ptr.?; + return ret.device_memory_ptr orelse { + log.err("[PJRT_Buffer_OpaqueDeviceMemoryDataPointer] plugin returned success with null device_memory_ptr", .{}); + return error.Internal; + }; } pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { @@ -1111,7 +1365,7 @@ pub const Buffer = opaque { pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), - .dst_memory = @ptrCast(@constCast(dst_memory)), + .dst_memory = dst_memory.inner(), }); return @ptrCast(ret.dst_buffer); } @@ -1134,6 +1388,11 @@ pub const Buffer = opaque { .buffer = self.inner(), }); } + + pub fn memoryLayout(self: *const Buffer, api: *const Api) ApiError!MemoryLayout { + const ret = try api.call(.PJRT_Buffer_GetMemoryLayout, .{ .buffer = self.inner() }); + return .fromCStruct(ret.layout); + } }; pub const Event = opaque { @@ -1162,7 +1421,8 @@ pub const Event = opaque { pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { if (self.isReady(api)) { - return; + const err = self.getEventError(api) orelse return; + return interpretPjrtError(api, err, "PJRT_Event_Await", .warn); } const Ctx = struct { @@ -1179,12 +1439,7 @@ pub const Event = opaque { }.call, &ctx); ctx.event.waitUncancelable(io); - if (ctx.err) |e| { - defer e.deinit(api); - const err_code = e.getCode(api).toApiError(); - log.err("{t} {s}", .{ err_code, e.getMessage(api) }); - return err_code; - } + if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady", .warn); } pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { @@ -1375,7 +1630,7 @@ pub const NamedValue = extern struct { }; pub fn kind(self: NamedValue) Kind { - return @enumFromInt(self.inner.type); + return @fromBackingInt(@intCast(self.inner.type)); } pub fn name(self: NamedValue) []const u8 { @@ -1392,14 +1647,14 @@ pub const NamedValue = extern struct { }; } - pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { + pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { return .{ .inner = .{ .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, .extension_start = null, .name = @ptrCast(name_), .name_size = name_.len, - .type = @intFromEnum(kind_), + .type = @backingInt(kind_), .unnamed_0 = switch (kind_) { .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, @@ -1512,3 +1767,98 @@ pub const Ffi = extern struct { } } }; + +pub const Profiler = struct { + pjrt_api: *const Api, + api: *const c.PLUGIN_Profiler_Api, + inner: *c.PLUGIN_Profiler, + + pub fn init(api_: *const Api, prof_ext: *const c.PJRT_Profiler_Extension, options_pb: []const u8) ApiError!Profiler { + var args: c.PLUGIN_Profiler_Create_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Create_Args), + .options = options_pb.ptr, + .options_size = options_pb.len, + .profiler = null, + }; + + const profiler_api: *const c.PLUGIN_Profiler_Api = @ptrCast(prof_ext.profiler_api); + if (profiler_api.create.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(api_).toApiError(); + } + + return .{ + .pjrt_api = api_, + .api = profiler_api, + .inner = args.profiler.?, + }; + } + + pub fn start(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Start_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Start_Args), + .profiler = self.inner, + }; + + if (self.api.start.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn stop(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Stop_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Stop_Args), + .profiler = self.inner, + }; + + if (self.api.stop.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) ![]u8 { + var args: c.PLUGIN_Profiler_CollectData_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_CollectData_Args), + .profiler = self.inner, + .buffer = null, + .buffer_size_in_bytes = 0, + }; + + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + if (args.buffer != null) { + var data = args.buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + return allocator.dupe(u8, data); + } + + if (args.buffer_size_in_bytes == 0) return &[_]u8{}; + + const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); + errdefer allocator.free(buffer); + + args.buffer = buffer.ptr; + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + // Trim trailing null byte logic + var data = buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + + const pb = try allocator.dupe(u8, data); + allocator.free(buffer); + + return pb; + } +}; diff --git a/platform_mappings b/platform_mappings deleted file mode 100644 index ed02329d..00000000 --- a/platform_mappings +++ /dev/null @@ -1,29 +0,0 @@ -platforms: - @zml//platforms:linux_amd64 - --cpu=k8 - - @zml//platforms:linux_arm64 - --cpu=aarch64 - - @zml//platforms:macos_arm64 - --cpu=darwin_arm64 - --apple_platform_type=macos - - @zml//platforms:macos_amd64 - --cpu=darwin_amd64 - --apple_platform_type=macos - -flags: - --cpu=darwin_arm64 - --apple_platform_type=macos - @zml//platforms:macos_arm64 - - --cpu=darwin_amd64 - --apple_platform_type=macos - @zml//platforms:macos_amd64 - - --cpu=k8 - @zml//platforms:linux_amd64 - - --cpu=aarch64 - @zml//platforms:linux_arm64 diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index a2a023b1..5ee2f176 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@rules_zig//zig:defs.bzl", "zig_library") +load("//bazel:zig_options.bzl", "zig_option", "zig_options") PLATFORMS = { "cpu": True, @@ -7,6 +8,9 @@ PLATFORMS = { "rocm": False, "tpu": False, "neuron": False, + "oneapi": False, + "metal": False, + "furiosa": False, } [ @@ -17,20 +21,75 @@ PLATFORMS = { for runtime, default in PLATFORMS.items() ] +[ + zig_option( + name = "{}_enabled".format(runtime), + bool = select({ + "//platforms:{}.enabled".format(runtime): True, + "//conditions:default": False, + }), + visibility = ["//platforms:__pkg__"], + ) + for runtime in PLATFORMS +] + +zig_options( + name = "options", + import_name = "platforms/options", + settings = [":{}_enabled".format(runtime) for runtime in PLATFORMS], + visibility = ["//platforms:__subpackages__"], +) + +# x86_64 and aarch64 [ config_setting( name = "{}.enabled".format(runtime), flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in PLATFORMS.keys() + for runtime in [ + "cpu", + "cuda", + ] ] +# x86_64 only +[ + config_setting( + name = "{}.enabled".format(runtime), + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//visibility:public"], + ) + for runtime in [ + "rocm", + "tpu", + "neuron", + "oneapi", + "furiosa", + ] +] + +config_setting( + name = "metal.enabled", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], + flag_values = {":metal": "True"}, + visibility = ["//visibility:public"], +) + zig_library( name = "platforms", main = "platforms.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ + ":options", "//pjrt", ] + [ "//platforms/{}".format(runtime) @@ -43,9 +102,11 @@ platform( constraint_values = [ "@platforms//cpu:x86_64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux + "--cpu=x86_64", ], visibility = ["//visibility:public"], ) @@ -55,9 +116,11 @@ platform( constraint_values = [ "@platforms//cpu:aarch64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux + "--cpu=aarch64", ], visibility = ["//visibility:public"], ) @@ -79,3 +142,26 @@ platform( ], visibility = ["//visibility:public"], ) + +constraint_setting(name = "execution_location") + +constraint_value( + name = "local_execution", + constraint_setting = ":execution_location", + visibility = ["//visibility:public"], +) + +# Special constrained host platform that toolchain can use as exec_compatible_with +# if they want to only be used when executing on the local machine. +# +# This is currently used by Zig toolchains so that they are only resolved for +# the host platform and thus can be used for no-remote-exec actions. +platform( + name = "local_execution_platform", + constraint_values = [":local_execution"], + exec_properties = { + "no-remote-exec": "1", + }, + parents = ["@platforms//host"], + visibility = ["//visibility:public"], +) diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 16e059d5..418c7837 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") @@ -18,42 +18,43 @@ config_setting( ], ) -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_CPU_DISABLED"], -) - copy_to_directory( name = "sandbox", srcs = select({ - ":darwin_arm64": ["@libpjrt_cpu_darwin_arm64//:libpjrt_cpu"], - ":darwin_amd64": ["@libpjrt_cpu_darwin_amd64//:libpjrt_cpu"], - "@platforms//os:linux": ["@libpjrt_cpu_linux_amd64//:libpjrt_cpu"], + ":darwin_arm64": ["@libzml_cpu_darwin_arm64//:libzml_cpu"], + ":darwin_amd64": ["@libzml_cpu_darwin_amd64//:libzml_cpu"], + "@platforms//os:linux": ["@libzml_cpu_linux_amd64//:libzml_cpu"], }), - out = "sandbox/lib", + out = "sandbox", include_external_repositories = ["**"], + replace_prefixes = { + "libzml_cpu_so": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, ) cc_library( - name = "libpjrt_cpu", + name = "libzml_cpu", data = [":sandbox"], - defines = ["ZML_RUNTIME_CPU"], + tags = ["manual"], ) zig_library( name = "cpu", import_name = "platforms/cpu", main = "cpu.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:cpu.enabled": [ - ":libpjrt_cpu", + ":libzml_cpu", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 35d0746e..08817afe 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -1,4 +1,5 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ @@ -6,47 +7,69 @@ package(default_visibility = ["//visibility:public"]) """ _BUILD_LINUX = "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "libzml_cpu_so", + src = "lib/libzml_cpu.so", + set_rpath = "$ORIGIN", + ), packages.filegroup( - name = "libpjrt_cpu", - srcs = ["libpjrt_cpu.so"], + name = "libzml_cpu", + srcs = [":libzml_cpu_so", "@llvm-libunwind1//:libunwind"], visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) _BUILD_DARWIN = packages.filegroup( - name = "libpjrt_cpu", - srcs = ["libpjrt_cpu.dylib"], + name = "libzml_cpu", + srcs = ["lib/libzml_cpu.dylib"], visibility = ["@zml//platforms/cpu:__subpackages__"], ) -def _cpu_pjrt_plugin_impl(mctx): +def _cpu_plugin_impl(mctx): + loaded_packages = packages.read(mctx, ["@zml//platforms/cpu:packages.lock.json"]) + pkg = loaded_packages["llvm-libunwind1"]["amd64"] + http_deb_archive( + name = "llvm-libunwind1", + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( + name = "libunwind", + srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + ), + ) + http_archive( - name = "libpjrt_cpu_linux_amd64", + name = "libzml_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "880a9575be4c950cd49b6ca20c2782869cea7ec445bac843ac55b21053b648ef", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_linux-amd64.tar.gz", + sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", ) http_archive( - name = "libpjrt_cpu_darwin_amd64", + name = "libzml_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "e1bd183a862fa1c504ade5aca1e000b273628ab2bd2a9eb2971067584934d882", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-amd64.tar.gz", + sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", ) http_archive( - name = "libpjrt_cpu_darwin_arm64", + name = "libzml_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "fb733aeec1284bfe4573acbe78ca38292ab521e8f207a3547d56d1a6d7173d47", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-arm64.tar.gz", + sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", + root_module_direct_deps = [ + "libzml_cpu_linux_amd64", + "libzml_cpu_darwin_amd64", + "libzml_cpu_darwin_arm64", + ], root_module_direct_dev_deps = [], ) -cpu_pjrt_plugin = module_extension( - implementation = _cpu_pjrt_plugin_impl, +cpu_plugin = module_extension( + implementation = _cpu_plugin_impl, ) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 37131347..523cd119 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -3,20 +3,20 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/cpu"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_CPU"); + return platforms_options.cpu_enabled; } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = io; // autofix - _ = allocator; // autofix + _ = io; + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } @@ -24,7 +24,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { + const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for CPU runtime", .{}); return error.FileNotFound; }; @@ -37,7 +37,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { }; var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cpu" ++ ext }); break :blk .loadFrom(path); }; } diff --git a/platforms/cpu/packages.lock.json b/platforms/cpu/packages.lock.json new file mode 100644 index 00000000..7e9963e3 --- /dev/null +++ b/platforms/cpu/packages.lock.json @@ -0,0 +1,65 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", + "name": "llvm-libunwind1", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" + ], + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/cpu/packages.yaml b/platforms/cpu/packages.yaml new file mode 100644 index 00000000..d6f97d05 --- /dev/null +++ b/platforms/cpu/packages.yaml @@ -0,0 +1,16 @@ +# +# bazel run @apt_cpu//:lock +# +version: 1 + +sources: + - channel: bookworm main + url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-22 main + url: https://apt.llvm.org/bookworm + +archs: + - "amd64" + +packages: + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 3bb7cfc0..7f7aed55 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -1,11 +1,17 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") + +zig_binary( + name = "compat_probe", + main = "compat_probe.zig", + visibility = ["//visibility:public"], +) zig_shared_library( name = "zmlxcuda", main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", - visibility = ["@libpjrt_cuda//:__subpackages__"], + visibility = ["//visibility:public"], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], @@ -13,37 +19,48 @@ zig_shared_library( ) cc_library( - name = "empty", - defines = ["ZML_RUNTIME_CUDA_DISABLED"], + name = "libzml_cuda_repo", + target_compatible_with = ["@platforms//os:linux"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], + "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], + }), ) cc_library( - name = "libpjrt_cuda", - hdrs = ["libpjrt_cuda.h"], - defines = ["ZML_RUNTIME_CUDA"], - deps = ["@libpjrt_cuda"], + name = "cuda_nvtx_headers", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvtx_linux_sbsa//:headers"], + }), +) + +cc_library( + name = "libzml_cuda", + hdrs = ["libzml_cuda.h"], + tags = ["manual"], + deps = [":libzml_cuda_repo"], ) zig_library( name = "cuda", - data = select({ - "//platforms:cuda.enabled": [ - "@flashattn//:libflashattn", - ], - "//conditions:default": [":empty"], - }), + srcs = ["compat_probe.zig"], import_name = "platforms/cuda", main = "cuda.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:cuda.enabled": [ - ":libpjrt_cuda", + ":libzml_cuda", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig new file mode 100644 index 00000000..3f890c6a --- /dev/null +++ b/platforms/cuda/compat_probe.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +pub const ExitCode = enum(u8) { + Success = 0, + CompatNotSupportedOnDevice = 1, + SystemDriverMismatch = 2, + UnexpectedError = 3, +}; + +const CUDA_SUCCESS = 0; +const CUDA_ERROR_SYSTEM_DRIVER_MISMATCH = 803; +const CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE = 804; + +pub fn main(init: std.process.Init) !u8 { + const arena = init.arena; + const io = init.io; + + const self_exe_dir = try std.process.executableDirPathAlloc(io, arena.allocator()); + const libcuda_path = try std.Io.Dir.path.joinZ(arena.allocator(), &.{ self_exe_dir, "..", "lib", "compat", "libcuda.so.1" }); + + var libcuda = try std.DynLib.open(libcuda_path); + const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; + const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; + + return @backingInt(switch (cuInit(0)) { + CUDA_SUCCESS => ExitCode.Success, + CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, + CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, + else => |v| blk: { + var err_str_ptr: [*c]const u8 = undefined; + if (cuGetErrorString(v, &err_str_ptr) == CUDA_SUCCESS) { + std.log.err("cuInit returned unexpected error code: {d}: {s}", .{ v, err_str_ptr }); + } else { + std.log.err("cuInit returned unexpected error code: {d}", .{v}); + } + break :blk ExitCode.UnexpectedError; + }, + }); +} diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 14e4867e..8482e52a 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -1,35 +1,49 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") +ARCHS = ["linux-x86_64", "linux-sbsa"] + +CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" +CUDA_VERSION = "13.3.0" +CUDA_VARIANT = "cuda13.3" +CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" + +CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" +CUDNN_VERSION = "9.24.0" +CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" + +NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" +NVSHMEM_VERSION = "3.6.5" +NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" + _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@zml//bazel:patchelf.bzl", "patchelf") """ -ARCH = "linux-x86_64" - -CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.1.1" -CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" - -CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.19.0" -CUDNN_REDIST_JSON_SHA256 = "6ad4f2c047ee03131d1efb95db56f78c24953cb30880a2edcf99407ff9362a04" - -NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.5.19" -NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" - -_UBUNTU_PACKAGES = { - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), -} +CUDA_COMPAT_FILES = [ + "libcuda.so.1", + "libcudadebugger.so.1", + "libnvidia-nvvm.so.4", + "libnvidia-nvvm70.so.4", + "libnvidia-ptxjitcompiler.so.1", +] CUDA_PACKAGES = { - "cuda_cudart": "\n".join([ + "cuda_nvml_dev": [ + packages.cc_library( + name = "nvml", + hdrs = ["include/nvml.h"], + includes = ["include"], + visibility = ["//visibility:public"], + ), + ], + "cuda_cudart": [ # Driver API only packages.cc_library( name = "cuda", @@ -41,34 +55,74 @@ CUDA_PACKAGES = { name = "cuda_cudart", srcs = ["lib/libcudart.so.13"], ), - ]), - "cuda_cupti": packages.filegroup( - name = "cuda_cupti", - srcs = ["lib/libcupti.so.13"], - ), - "cuda_nvtx": "\n".join([ + ], + "cuda_cupti": [ + packages.filegroup( + name = "cuda_cupti", + srcs = ["lib/libcupti.so.13"], + ), + ], + "cuda_nvtx": [ + packages.cc_library_hdrs_glob( + name = "headers", + hdrs_glob = ["include/nvtx3/**"], + includes = ["include"], + ), packages.filegroup( name = "cuda_nvtx", srcs = ["lib/libnvtx3interop.so"], ), - ]), - "libcufft": packages.filegroup( - name = "libcufft", - srcs = ["lib/libcufft.so.12"], - ), - "libcusolver": packages.filegroup( - name = "libcusolver", - srcs = ["lib/libcusolver.so.12"], - ), - "libcusparse": packages.filegroup( - name = "libcusparse", - srcs = ["lib/libcusparse.so.12"], - ), - "libnvjitlink": packages.filegroup( - name = "libnvjitlink", - srcs = ["lib/libnvJitLink.so.13"], - ), - "cuda_nvcc": "\n".join([ + ], + "cuda_compat": [ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ + """filegroup( + name = "cuda_compat", + srcs = [ + "compat/libnvidia-gpucomp.so.610.43.02", + "compat/libnvidia-tileiras.so.610.43.02", + ] + {} + select({{ + "@llvm//platforms/config:linux_x86_64": [ + "compat/libnvidia-pkcs11-openssl3.so.610.43.02", + ], + "@llvm//platforms/config:linux_aarch64": [], + }}), + )""".format(repr([ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ])), + ], + "libcufft": [ + packages.filegroup( + name = "libcufft", + srcs = ["lib/libcufft.so.12"], + ), + ], + "libcusolver": [ + packages.filegroup( + name = "libcusolver", + srcs = ["lib/libcusolver.so.12"], + ), + ], + "libcusparse": [ + packages.filegroup( + name = "libcusparse", + srcs = ["lib/libcusparse.so.12"], + ), + ], + "libnvjitlink": [ + packages.filegroup( + name = "libnvjitlink", + srcs = ["lib/libnvJitLink.so.13"], + ), + ], + "cuda_nvcc": [ packages.filegroup( name = "cuda_nvcc", srcs = [ @@ -76,34 +130,35 @@ CUDA_PACKAGES = { "bin/nvlink", ], ), - packages.cc_import( - name = "nvptxcompiler", - static_library = "lib/libnvptxcompiler_static.a", + ], + "cuda_tileiras": [ + packages.filegroup( + name = "cuda_tileiras", + srcs = ["bin/tileiras"], ), - ]), - "libnvvm": "\n".join([ + ], + "libnvvm": [ packages.filegroup( name = "libnvvm", srcs = [ "nvvm/bin/cicc", "nvvm/libdevice/libdevice.10.bc", + "nvvm/lib64/libnvvm.so", + "nvvm/lib64/libnvvm.so.4", + "nvvm/lib64/libnvvm.so.4.0.0", ], ), - packages.cc_import( - name = "nvptxcompiler", - static_library = "lib/libnvptxcompiler_static.a", - ), - ]), - "cuda_nvrtc": "\n".join([ + ], + "cuda_nvrtc": [ packages.filegroup( name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.1", + "lib/libnvrtc-builtins.so.13.3", ], ), - ]), - "libcublas": "\n".join([ + ], + "libcublas": [ packages.filegroup( name = "libcublas", srcs = [ @@ -111,11 +166,11 @@ CUDA_PACKAGES = { "lib/libcublas.so.13", ], ), - ]), + ], } CUDNN_PACKAGES = { - "cudnn": "\n".join([ + "cudnn": [ packages.filegroup( name = "cudnn", srcs = [ @@ -126,23 +181,75 @@ CUDNN_PACKAGES = { "lib/libcudnn_graph.so.9", "lib/libcudnn_engines_precompiled.so.9", "lib/libcudnn_engines_runtime_compiled.so.9", + "lib/libcudnn_engines_tensor_ir.so.9", "lib/libcudnn_heuristic.so.9", ], ), - ]), + ], } NVSHMEM_PACKAGES = { - "libnvshmem": packages.filegroup( - name = "libnvshmem", - srcs = [ - "lib/libnvshmem_host.so.3", - "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.4", - ], - ), + "libnvshmem": [ + packages.filegroup( + name = "libnvshmem", + srcs = [ + "lib/libnvshmem_host.so.3", + "lib/nvshmem_bootstrap_uid.so.3", + "lib/nvshmem_transport_ibrc.so.5", + ], + ), + ], } +_UBUNTU_PACKAGES = { + "llvm-libunwind1": [ + """filegroup( + name = "libunwind", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + "@llvm//platforms/config:linux_aarch64": ["usr/lib/aarch64-linux-gnu/libunwind.so.1"], + }), + )""", + ], + "zlib1g": [ + """filegroup( + name = "zlib1g", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["lib/x86_64-linux-gnu/libz.so.1"], + "@llvm//platforms/config:linux_aarch64": ["lib/aarch64-linux-gnu/libz.so.1"], + }), + )""", + ], +} + +_PLUGINS = { + "amd64": { + "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", + }, + "arm64": { + "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", + }, +} + +_NCCL_ASSETS = { + "amd64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", + "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", + }, + "arm64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", + "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", + }, +} + +def _repo_suffix(arch): + return "linux_{}".format(arch) + +def _repo_name(name, arch): + return "{}_{}".format(name, _repo_suffix(arch)) + def _read_redist_json(mctx, url, sha256): fname = ".{}.json".format(sha256) mctx.download( @@ -156,6 +263,7 @@ def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//platforms/cuda:packages.lock.json", ]) + CUDA_REDIST = _read_redist_json( mctx, url = CUDA_REDIST_PREFIX + "redistrib_{}.json".format(CUDA_VERSION), @@ -174,77 +282,100 @@ def _cuda_impl(mctx): sha256 = CUDNN_REDIST_JSON_SHA256, ) + #TODO(cerisier): for each architecture for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] - http_deb_archive( - name = pkg_name, - urls = pkg["urls"], - sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - ) + pkgs = loaded_packages[pkg_name] + for arch, pkg in pkgs.items(): + http_deb_archive( + name = _repo_name(pkg_name, arch), + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + ) for pkg, build_file_content in CUDA_PACKAGES.items(): pkg_data = CUDA_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = CUDA_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDA_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in CUDNN_PACKAGES.items(): - pkg_data = CUDNN_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get("cuda13", arch_data) - http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = CUDNN_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in NVSHMEM_PACKAGES.items(): - pkg_data = NVSHMEM_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get("cuda13", arch_data) + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = NVSHMEM_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + + for arch, arch_config in _NCCL_ASSETS.items(): http_archive( - name = pkg, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + name = _repo_name("nccl", arch), + urls = [arch_config["url"]], + type = "zip", + sha256 = arch_config["sha256"], + build_file_content = "\n".join([ + _BUILD_FILE_DEFAULT_VISIBILITY, + packages.filegroup( + name = "nccl", + srcs = ["nvidia/nccl/lib/libnccl.so.2"], + ), + ]), ) - http_archive( - name = "nccl", - urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl"], - type = "zip", - sha256 = "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( - name = "nccl", - srcs = ["nvidia/nccl/lib/libnccl.so.2"], - ), - ) - - http_archive( - name = "libpjrt_cuda", - build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "ebc5f0fa54d38ec85346f0f02b29a6497ee394b9e5fa4832da5db16d3296ae84", - ) + for arch, arch_config in _PLUGINS.items(): + http_archive( + name = _repo_name("libzml_cuda", arch), + build_file = "libzml_cuda.BUILD.bazel", + url = arch_config["url"], + sha256 = arch_config["sha256"], + ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_cuda"], + root_module_direct_deps = [ + "libzml_cuda_linux_amd64", + "libzml_cuda_linux_arm64", + "cuda_nvml_dev_linux_x86_64", + "cuda_nvml_dev_linux_sbsa", + "zlib1g_linux_arm64", + "zlib1g_linux_amd64", + "cuda_nvtx_linux_x86_64", + "cuda_nvtx_linux_sbsa", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index b0ff4425..692ea226 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -5,24 +5,71 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); +const compat_probe = @import("compat_probe.zig"); + const nvidiaLibsPath = "/cuda/"; const log = std.log.scoped(.@"zml/platforms/cuda"); +fn findCudaSandbox( + r: anytype, + buffer: *[std.Io.Dir.max_path_bytes]u8, +) !?[]const u8 { + const candidate = switch (builtin.cpu.arch) { + .aarch64 => "libzml_cuda_linux_arm64/sandbox", + .x86_64 => "libzml_cuda_linux_amd64/sandbox", + else => return null, + }; + return try r.rlocation(candidate, buffer); +} + pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_CUDA"); + return platforms_options.cuda_enabled; +} + +pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nvidia_compat_path = try stdx.Io.Dir.path.bufJoinZ(&buf, &.{ sandbox_path, "bin", "compat_probe" }); + + var child = try std.process.spawn(io, .{ + .argv = &[_][]const u8{nvidia_compat_path}, + .cwd = .{ .path = sandbox_path }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + }); + defer child.kill(io); + + const res = child.wait(io) catch |err| { + log.err("Failed to run CUDA compatibility probe: {any}", .{err}); + return err; + }; + const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); + + return switch (result) { + .Success => true, + .SystemDriverMismatch, .CompatNotSupportedOnDevice => false, + .UnexpectedError => blk: { + log.err("CUDA compatibility probe returned unexpected error code", .{}); + break :blk false; + }, + }; } fn hasNvidiaDevice(io: std.Io) bool { - std.Io.Dir.accessAbsolute(io, "/dev/nvidiactl", .{ .read = true }) catch return false; - return true; + for (&[_][]const u8{ "/dev/nvidiactl", "/dev/dxg" }) |dev| { + std.Io.Dir.accessAbsolute(io, dev, .{ .read = true }) catch continue; + return true; + } + return false; } fn hasCudaPathInLDPath() bool { const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; - return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; + return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { @@ -51,7 +98,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { + const sandbox_path = try findCudaSandbox(r, &path_buf) orelse { log.err("Failed to find sandbox path for CUDA runtime", .{}); return error.FileNotFound; }; @@ -61,17 +108,25 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); { - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); - _ = std.c.dlopen(path, .{ .NOW = true, .GLOBAL = true }) orelse { - log.err("Unable to dlopen libnvtx3interop.so: {s}", .{std.c.dlerror().?}); - return error.DlError; + const cudaCompat = needsCudaCompat(io, sandbox_path) catch |err| blk: { + log.err("Unable to determine wether or not to use CUDA Compat, disabling: {any}", .{err}); + break :blk false; }; + + if (cudaCompat) { + log.info("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "compat", "libcuda.so.1" }); + _ = std.c.dlopen(path, .{ .NOW = true }) orelse { + log.warn("Failed to load CUDA compatibility library from {s}: {any}", .{ path, std.mem.span(std.c.dlerror()) }); + }; + log.info("Loaded CUDA compatibility libraries.", .{}); + } } return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cuda.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel deleted file mode 100644 index 9ef2bbf8..00000000 --- a/platforms/cuda/flashattn/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "flashattn", - import_name = "platforms/cuda/flashattn", - main = "flashattn.zig", - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "@flashattn//:c", - "@rules_zig//zig/runfiles", - ], -) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig deleted file mode 100644 index 2a743d33..00000000 --- a/platforms/cuda/flashattn/flashattn.zig +++ /dev/null @@ -1,112 +0,0 @@ -const std = @import("std"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); -pub const FA2MhaFwdParams = c.FA2MhaFwdParams; -pub const FA2MhaVarlenFwdParams = c.FA2MhaVarlenFwdParams; -pub const FA3MhaFwdParams = c.FA3MhaFwdParams; -const runfiles = @import("runfiles"); - -pub const Tensor = struct { - inner: c.FlashattnTensor, - - pub fn init(ptr: *anyopaque, dims: []const i64, strides: []const i64, dtype: DataType) Tensor { - const rank = dims.len; - var ret: Tensor = .{ - .inner = .{ - .ptr = ptr, - .rank = @intCast(rank), - .dims = undefined, - .strides = undefined, - .dtype = @intCast(@intFromEnum(dtype)), - }, - }; - @memcpy(ret.inner.dims[0..rank], dims); - @memcpy(ret.inner.strides[0..rank], strides); - return ret; - } -}; - -pub const DataType = enum(c_int) { - f32 = c.CAPI_FLOAT, - f16 = c.CAPI_FLOAT16, - bf16 = c.CAPI_BFLOAT16, - i32 = c.CAPI_INT32, - i8 = c.CAPI_INT8, -}; - -const Fa2MhaFwdFunc = *const fn ( - q: *const Tensor, - k: *const Tensor, - v: *const Tensor, - out: *const Tensor, - softmax_lse: ?*const Tensor, - alibi_slopes_: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - params: ?*const FA2MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa2MhaVarlenFwdFunc = *const fn ( - q: *const Tensor, - k: *const Tensor, - v: *const Tensor, - out: *const Tensor, - cu_seqlens_q: ?*const Tensor, - cu_seqlens_k: ?*const Tensor, - seqused_k: ?*const Tensor, - block_table: ?*const Tensor, - softmax_lse: ?*const Tensor, - alibi_slopes_: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - params: ?*const FA2MhaVarlenFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -const Fa3MhaFwdFunc = *const fn ( - q: ?*const Tensor, - k: ?*const Tensor, - v: ?*const Tensor, - out: ?*const Tensor, - cu_seqlens_q: ?*const Tensor, - cu_seqlens_k: ?*const Tensor, - seqused_q: ?*const Tensor, - seqused_k: ?*const Tensor, - page_table: ?*const Tensor, - q_descale: ?*const Tensor, - k_descale: ?*const Tensor, - v_descale: ?*const Tensor, - softmax_lse: ?*const Tensor, - softmax_lse_accum: ?*const Tensor, - out_accum: ?*const Tensor, - scheduler_metadata: ?*const Tensor, - s_aux: ?*const Tensor, - cp_tot_seqused_k: ?*const Tensor, - params: ?*const FA3MhaFwdParams, - stream: ?*anyopaque, -) callconv(.c) void; - -pub var fa2_mha_fwd: Fa2MhaFwdFunc = undefined; -pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; -pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - _ = io; // autofix - _ = allocator; // autofix - - const r = try bazel.runfiles(bazel_builtin.current_repository); - - var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; - var lib = std.DynLib.open(library) catch |err| { - std.log.err("Failed to open libflashattn.so: {any}", .{err}); - return err; - }; - - fa2_mha_varlen_fwd = lib.lookup(Fa2MhaVarlenFwdFunc, "fa2_mha_varlen_fwd") orelse return error.NotFound; - fa2_mha_fwd = lib.lookup(Fa2MhaFwdFunc, "fa2_mha_fwd") orelse return error.NotFound; - fa3_mha_fwd = lib.lookup(Fa3MhaFwdFunc, "fa3_mha_fwd") orelse return error.NotFound; -} diff --git a/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..8a61e394 --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,48 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +_DEPS = [ + "//bazel", + "@rules_zig//zig/runfiles", +] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:c", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:c", + ], + "//conditions:default": [], +}) + +_DATA = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:libraries", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:libraries", + ], + "//conditions:default": [], +}) + +zig_library( + name = "flashinfer_cutlass_moe", + data = _DATA, + import_name = "platforms/cuda/flashinfer_cutlass_moe", + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], + deps = _DEPS, +) + +zig_test( + name = "test", + data = _DATA, + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = _DEPS, +) diff --git a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig new file mode 100644 index 00000000..0b4360cf --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const builtin = @import("builtin"); +pub const c = @import("c"); + +pub const Status = c.zml_fi_cutlass_moe_status; +pub const DType = c.zml_fi_cutlass_moe_dtype; +pub const Activation = c.zml_fi_cutlass_moe_activation; +pub const RunnerOptions = c.zml_fi_cutlass_moe_runner_options; +pub const Context = c.zml_fi_cutlass_moe_context; +pub const Io = c.zml_fi_cutlass_moe_io; +pub const Workspace = c.zml_fi_cutlass_moe_workspace; +pub const WorkspaceRequirements = c.zml_fi_cutlass_moe_workspace_requirements; +pub const Runner = c.zml_fi_cutlass_moe_runner; + +pub const abiVersionExpected = c.ZML_FI_CUTLASS_MOE_ABI_VERSION; + +const AbiVersionFn = @TypeOf(&c.zml_fi_cutlass_moe_abi_version); +const CompiledSmFn = @TypeOf(&c.zml_fi_cutlass_moe_compiled_sm); +const LastErrorFn = @TypeOf(&c.zml_fi_cutlass_moe_last_error); +const DeviceIsSupportedFn = @TypeOf(&c.zml_fi_cutlass_moe_device_is_supported); +const RunnerCreateFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_create); +const RunnerDestroyFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_destroy); +const GetTacticCountsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_counts); +const GetTacticOccupancyFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_occupancy); +const GetWorkspaceRequirementsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_workspace_requirements); +const RunFn = @TypeOf(&c.zml_fi_cutlass_moe_run); + +pub const Api = struct { + library: std.DynLib, + abiVersion: AbiVersionFn, + compiledSm: CompiledSmFn, + lastError: LastErrorFn, + deviceIsSupported: DeviceIsSupportedFn, + runnerCreate: RunnerCreateFn, + runnerDestroy: RunnerDestroyFn, + getTacticCounts: GetTacticCountsFn, + getTacticOccupancy: GetTacticOccupancyFn, + getWorkspaceRequirements: GetWorkspaceRequirementsFn, + run: RunFn, +}; + +var loaded_api: ?Api = null; + +fn libraryRunfile(sm: u16, buffer: []u8) ![]const u8 { + if (comptime builtin.os.tag != .linux) return error.UnsupportedPlatform; + const cpu_arch = comptime switch (builtin.cpu.arch) { + .aarch64 => "arm64", + .x86_64 => "amd64", + else => return error.UnsupportedPlatform, + }; + return try std.fmt.bufPrint( + buffer, + "flashinfer_cutlass_moe_linux_{s}/lib/libflashinfer_cutlass_moe_sm{d}.so", + .{ cpu_arch, sm }, + ); +} + +fn loadApi(path: []const u8) !Api { + var library = std.DynLib.open(path) catch |err| { + std.log.err("Failed to open FlashInfer CUTLASS MoE library {s}: {any}", .{ path, err }); + return err; + }; + errdefer library.close(); + + const api: Api = .{ + .library = library, + .abiVersion = library.lookup(AbiVersionFn, "zml_fi_cutlass_moe_abi_version") orelse + return error.SymbolNotFound, + .compiledSm = library.lookup(CompiledSmFn, "zml_fi_cutlass_moe_compiled_sm") orelse + return error.SymbolNotFound, + .lastError = library.lookup(LastErrorFn, "zml_fi_cutlass_moe_last_error") orelse + return error.SymbolNotFound, + .deviceIsSupported = library.lookup(DeviceIsSupportedFn, "zml_fi_cutlass_moe_device_is_supported") orelse + return error.SymbolNotFound, + .runnerCreate = library.lookup(RunnerCreateFn, "zml_fi_cutlass_moe_runner_create") orelse + return error.SymbolNotFound, + .runnerDestroy = library.lookup(RunnerDestroyFn, "zml_fi_cutlass_moe_runner_destroy") orelse + return error.SymbolNotFound, + .getTacticCounts = library.lookup(GetTacticCountsFn, "zml_fi_cutlass_moe_get_tactic_counts") orelse + return error.SymbolNotFound, + .getTacticOccupancy = library.lookup(GetTacticOccupancyFn, "zml_fi_cutlass_moe_get_tactic_occupancy") orelse + return error.SymbolNotFound, + .getWorkspaceRequirements = library.lookup( + GetWorkspaceRequirementsFn, + "zml_fi_cutlass_moe_get_workspace_requirements", + ) orelse return error.SymbolNotFound, + .run = library.lookup(RunFn, "zml_fi_cutlass_moe_run") orelse + return error.SymbolNotFound, + }; + if (api.abiVersion() != abiVersionExpected) return error.AbiVersionMismatch; + return api; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + _ = allocator; + if (loaded_api) |*api| { + if (api.compiledSm() != sm) return error.ArchitectureAlreadyLoaded; + return; + } + + const runfiles = try bazel.runfiles(bazel_builtin.current_repository); + var runfile_name_buffer: [160]u8 = undefined; + const runfile_name = try libraryRunfile(sm, &runfile_name_buffer); + var runfile_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const runfile_path = (try runfiles.rlocation(runfile_name, &runfile_path_buffer)) orelse + return error.NotFound; + + // Preserve the producer library's $ORIGIN-relative dependency lookup when + // the development repository exposes it through Bazel symlinks. + var canonical_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const canonical_length = if (std.fs.path.isAbsolute(runfile_path)) + try std.Io.Dir.realPathFileAbsolute(io, runfile_path, &canonical_buffer) + else + try std.Io.Dir.cwd().realPathFile(io, runfile_path, &canonical_buffer); + var api = try loadApi(canonical_buffer[0..canonical_length]); + errdefer api.library.close(); + if (api.compiledSm() != sm) return error.WrongArchitectureLibrary; + loaded_api = api; +} + +pub fn apiForDevice(device: i32) !*Api { + const api = if (loaded_api) |*value| value else return error.BackendNotLoaded; + var supported: u8 = 0; + const status = api.deviceIsSupported(device, &supported); + if (status != c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) { + if (api.lastError()) |message| { + std.log.err("FlashInfer CUTLASS MoE device probe failed: {s}", .{std.mem.span(message)}); + } + return error.DeviceProbeFailed; + } + if (supported != 0) return api; + return error.UnsupportedArchitecture; +} + +test "select host and GPU architecture-specific FlashInfer CUTLASS MoE library" { + const test_case = &.{ + .{ + .compat_version = 120, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", + }, + }, + .{ + .compat_version = 103, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm103.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm103.so", + }, + }, + }; + + inline for (test_case) |t| { + var buffer: [160]u8 = undefined; + const path = try libraryRunfile(t.compat_version, &buffer); + try std.testing.expectEqualStrings(t.expected, path); + + try load(std.testing.allocator, std.testing.io, 120); + const api = if (loaded_api) |*value| value else return error.NotLoaded; + try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); + } +} diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel deleted file mode 100644 index 372874a9..00000000 --- a/platforms/cuda/libpjrt_cuda.BUILD.bazel +++ /dev/null @@ -1,70 +0,0 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:patchelf.bzl", "patchelf") -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -patchelf( - name = "libpjrt_cuda_so", - src = "libpjrt_cuda.so", - add_needed = [ - "libzmlxcuda.so.0", - ], - rename_dynamic_symbols = { - "dlopen": "zmlxcuda_dlopen", - }, - replace_needed = { - "nvshmem_transport_ibrc.so.3": "nvshmem_transport_ibrc.so.4", - "libnvrtc-builtins.so.13.0": "libnvrtc-builtins.so.13.1", - }, - set_rpath = "$ORIGIN", -) - -copy_to_directory( - name = "sandbox", - srcs = [ - ":libpjrt_cuda_so", - "@cuda_cudart", - "@cuda_cupti", - "@cuda_nvcc", - "@libnvvm", - "@cuda_nvrtc", - "@cuda_nvtx", - "@cudnn", - "@libnvshmem", - "@libcublas", - "@libcufft", - "@libcusolver", - "@libcusparse", - "@libnvjitlink", - "@nccl", - "@zlib1g", - "@zml//platforms/cuda:zmlxcuda", - ], - replace_prefixes = { - "nvidia/nccl/lib": "lib", - "nvvm/lib64": "lib", - "libpjrt_cuda_so": "lib", - "lib/x86_64-linux-gnu": "lib", - "platforms/cuda": "lib", - }, - add_directory_to_runfiles = False, - include_external_repositories = ["**"], -) - -cc_library( - name = "libpjrt_cuda", - data = [":sandbox"], - deps = [ - "@cuda_cudart//:cuda", - ], - linkopts = [ - # Defer function call resolution until the function is called - # (lazy loading) rather than at load time. - # - # This is required because we want to let downstream use weak CUDA symbols. - # - # We force it here because -z,now (which resolve all symbols at load time), - # is the default in most bazel CC toolchains as well as in certain linkers. - "-Wl,-z,lazy", - ], - visibility = ["@zml//platforms/cuda:__subpackages__"], -) diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel new file mode 100644 index 00000000..e79ab900 --- /dev/null +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -0,0 +1,108 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//bazel:patchelf.bzl", "patchelf") +load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") + +patchelf( + name = "libzml_cuda_so", + src = "lib/libzml_cuda.so", + add_needed = [ + "libzmlxcuda.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxcuda_dlopen", + }, + set_rpath = "$ORIGIN", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/cutlass_ir.so", + ":libzml_cuda_so", + "@zml//platforms/cuda:compat_probe", + "@zml//platforms/cuda:zmlxcuda", + ] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_compat_linux_x86_64//:cuda_compat", + "@cuda_cudart_linux_x86_64//:cuda_cudart", + "@cuda_cupti_linux_x86_64//:cuda_cupti", + "@cuda_nvcc_linux_x86_64//:cuda_nvcc", + "@cuda_nvrtc_linux_x86_64//:cuda_nvrtc", + "@cuda_nvtx_linux_x86_64//:cuda_nvtx", + "@cuda_tileiras_linux_x86_64//:cuda_tileiras", + "@cudnn_linux_x86_64//:cudnn", + "@libcublas_linux_x86_64//:libcublas", + "@libcufft_linux_x86_64//:libcufft", + "@libcusolver_linux_x86_64//:libcusolver", + "@libcusparse_linux_x86_64//:libcusparse", + "@libnvjitlink_linux_x86_64//:libnvjitlink", + "@libnvshmem_linux_x86_64//:libnvshmem", + "@libnvvm_linux_x86_64//:libnvvm", + "@llvm-libunwind1_linux_amd64//:libunwind", + "@nccl_linux_amd64//:nccl", + "@zlib1g_linux_amd64//:zlib1g", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_compat_linux_sbsa//:cuda_compat", + "@cuda_cudart_linux_sbsa//:cuda_cudart", + "@cuda_cupti_linux_sbsa//:cuda_cupti", + "@cuda_nvcc_linux_sbsa//:cuda_nvcc", + "@cuda_nvrtc_linux_sbsa//:cuda_nvrtc", + "@cuda_nvtx_linux_sbsa//:cuda_nvtx", + "@cuda_tileiras_linux_sbsa//:cuda_tileiras", + "@cudnn_linux_sbsa//:cudnn", + "@libcublas_linux_sbsa//:libcublas", + "@libcufft_linux_sbsa//:libcufft", + "@libcusolver_linux_sbsa//:libcusolver", + "@libcusparse_linux_sbsa//:libcusparse", + "@libnvjitlink_linux_sbsa//:libnvjitlink", + "@libnvshmem_linux_sbsa//:libnvshmem", + "@libnvvm_linux_sbsa//:libnvvm", + "@llvm-libunwind1_linux_arm64//:libunwind", + "@nccl_linux_arm64//:nccl", + "@zlib1g_linux_arm64//:zlib1g", + ], + }), + add_directory_to_runfiles = False, + include_external_repositories = ["**"], + replace_prefixes = { + "compat": "lib/compat", + "nvidia/nccl/lib": "lib", + "nvvm/lib64": "lib", + "libzml_cuda_so": "lib", + "lib/x86_64-linux-gnu": "lib", + "lib/aarch64-linux-gnu": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + "usr/lib/aarch64-linux-gnu": "lib", + "platforms/cuda/compat_probe": "bin/compat_probe", + "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", + } | { + "{}.patchelf".format(file): "lib/compat" + for file in CUDA_COMPAT_FILES + }, +) + +cc_library( + name = "libzml_cuda", + data = [":sandbox"], + linkopts = [ + # Defer function call resolution until the function is called + # (lazy loading) rather than at load time. + # + # This is required because we want to let downstream use weak CUDA symbols. + # + # We force it here because -z,now (which resolve all symbols at load time), + # is the default in most bazel CC toolchains as well as in certain linkers. + "-Wl,-z,lazy", + ], + visibility = ["@zml//platforms/cuda:__subpackages__"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_cudart_linux_x86_64//:cuda", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_cudart_linux_sbsa//:cuda", + ], + }), +) diff --git a/platforms/cuda/libpjrt_cuda.h b/platforms/cuda/libzml_cuda.h similarity index 100% rename from platforms/cuda/libpjrt_cuda.h rename to platforms/cuda/libzml_cuda.h diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index b80a2c36..3563bc61 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "zlib1g_1-1.2.13.dfsg-1_amd64", - "name": "zlib1g", - "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", + "name": "llvm-libunwind1", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" ], - "version": "1:1.2.13.dfsg-1" + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" }, { "arch": "amd64", @@ -59,6 +59,228 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" ], "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_amd64", + "name": "cuda-compat-13-4", + "sha256": "1b814aa705b4dfe955420c967bd545f39ef7b67f32bd0f285a005ea296b02831", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./cuda-compat-13-4_615.71.09-2ubuntu1_amd64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_amd64", + "name": "libnccl2", + "sha256": "55a3e32a3f560f20e2f423cd38f1f284951e92093d1e8fbaabb4c42ac76f3195", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./libnccl2_2.32.3-1+cuda13.4_amd64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_arm64", + "name": "llvm-libunwind1", + "sha256": "05890076ab166d0fe3eab3ab08f9076bca1edc7322d123df8970760ae26c25e8", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_arm64.deb" + ], + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "sha256": "9b25c7c2aaa9328b2cfd92b47cc48b24631bfdff0c61afe05cba937911fcf239", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_arm64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "sha256": "576926b283613db80168ddf76380a3bd877602778cf0d226caa7bfbfa71eacf3", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "sha256": "674cf6cba6d432bd200c45fe866c1652c7a53523cc2e7a613e05bc4abf7b5440", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_arm64", + "name": "zlib1g", + "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_arm64", + "name": "cuda-compat-13-4", + "sha256": "83cdca90db73a36784a077693e3b48565d852158c63f604194c1f0d42c2e5458", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./cuda-compat-13-4_615.71.09-2ubuntu1_arm64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_arm64", + "name": "libnccl2", + "sha256": "809a6eea2f8a570050229bc218e30d4280829cbe01e824e052c96ee9203a7fe5", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./libnccl2_2.32.3-1+cuda13.4_arm64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "sha256": "26e138c677a985775331373828a6c286c551ff397cb735d00e2383cb273d1cb2", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" } ], "version": 1 diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index 4b3b04c0..010ca572 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -6,9 +6,21 @@ version: 1 sources: - channel: bookworm main url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-22 main + url: https://apt.llvm.org/bookworm + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 + archs: [amd64] + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa + archs: [arm64] archs: - - "amd64" + - "amd64" + - "arm64" packages: - - "zlib1g" + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. + - "zlib1g" + - "cuda-compat-13-4" + - "libnccl2" diff --git a/platforms/furiosa/BUILD.bazel b/platforms/furiosa/BUILD.bazel new file mode 100644 index 00000000..3118e6d0 --- /dev/null +++ b/platforms/furiosa/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("@rules_zig//zig:defs.bzl", "zig_library") + +exports_files(["libzml_furiosa.BUILD.bazel"]) + +zig_library( + name = "furiosa", + import_name = "platforms/furiosa", + main = "furiosa.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + ] + select({ + "//platforms:furiosa.enabled": ["@libzml_furiosa"], + "//conditions:default": [], + }), +) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl new file mode 100644 index 00000000..afb26864 --- /dev/null +++ b/platforms/furiosa/furiosa.bzl @@ -0,0 +1,21 @@ +"""Consume the complete Furiosa sandbox produced by XLA.""" + +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") + +def _furiosa_impl(mctx): + http_archive( + name = "libzml_furiosa", + build_file = "libzml_furiosa.BUILD.bazel", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libzml_furiosa"], + root_module_direct_dev_deps = [], + ) + +furiosa_packages = module_extension( + implementation = _furiosa_impl, +) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig new file mode 100644 index 00000000..ee33fd58 --- /dev/null +++ b/platforms/furiosa/furiosa.zig @@ -0,0 +1,57 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/furiosa"); + +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; + +fn setEnv(name: [*:0]const u8, value: [:0]const u8, overwrite: c_int) !void { + if (setenv(name, value, overwrite) != 0) return error.SetEnvFailed; +} + +pub fn isEnabled() bool { + return platforms_options.furiosa_enabled; +} + +fn hasFuriosaDevice(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/rngd", .{ .read = true }) catch return false; + return true; +} + +fn hasDmaHeapAccess(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/dma_heap/system", .{ .read = true }) catch return false; + return true; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled() or builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return error.Unavailable; + if (!hasFuriosaDevice(io)) { + return error.Unavailable; + } + if (!hasDmaHeapAccess(io)) { + std.debug.panic("Furiosa needs read access to /dev/dma_heap/system for pinned host memory", .{}); + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox = try r.rlocation("libzml_furiosa/sandbox", &path_buf) orelse { + log.err("Missing Furiosa sandbox runfile", .{}); + return error.FileNotFound; + }; + + const root = try std.Io.Dir.cwd().realPathFileAlloc(io, sandbox, allocator); + defer allocator.free(root); + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + + const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); + log.info("Loading Furiosa plugin: {s}", .{library}); + return .loadFrom(library); +} diff --git a/platforms/furiosa/libzml_furiosa.BUILD.bazel b/platforms/furiosa/libzml_furiosa.BUILD.bazel new file mode 100644 index 00000000..3a2da19a --- /dev/null +++ b/platforms/furiosa/libzml_furiosa.BUILD.bazel @@ -0,0 +1,23 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +# XLA owns the full payload. ZML only places it in the platform's runfiles. +copy_to_directory( + name = "sandbox", + srcs = glob( + [ + "bin/**", + "include/**", + "lib/**", + ], + allow_empty = False, + ), + add_directory_to_runfiles = True, + visibility = ["//visibility:public"], +) + +cc_library( + name = "libzml_furiosa", + data = [":sandbox"], + visibility = ["//visibility:public"], +) diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel new file mode 100644 index 00000000..48bca771 --- /dev/null +++ b/platforms/metal/BUILD.bazel @@ -0,0 +1,45 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +copy_to_directory( + name = "metal_toolchain", + srcs = [ + "@metal_toolchain//:metal_compile_runtime", + "@metal_toolchain//:metal_compile_tools", + ], + out = "metal_toolchain", + include_external_repositories = ["**"], + replace_prefixes = { + "Metal.xctoolchain/usr/metal/32023": ".", + }, + target_compatible_with = ["@platforms//os:macos"], +) + +cc_library( + name = "libzml_metal", + hdrs = ["metal.h"], + data = [":metal_toolchain"], + tags = ["manual"], + deps = ["@libzml_metal"], +) + +zig_library( + name = "metal", + import_name = "platforms/metal", + main = "metal.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:metal.enabled": [ + ":libzml_metal", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/metal/libzml_metal.BUILD.bazel b/platforms/metal/libzml_metal.BUILD.bazel new file mode 100644 index 00000000..7bac767d --- /dev/null +++ b/platforms/metal/libzml_metal.BUILD.bazel @@ -0,0 +1,17 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/libzml_metal.dylib", + ], + add_directory_to_runfiles = False, + include_external_repositories = ["**"], +) + +cc_library( + name = "libzml_metal", + data = [":sandbox"], + visibility = ["@zml//platforms/metal:__subpackages__"], +) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl new file mode 100644 index 00000000..71e1858a --- /dev/null +++ b/platforms/metal/metal.bzl @@ -0,0 +1,19 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _metal_impl(mctx): + http_archive( + name = "libzml_metal", + build_file = "libzml_metal.BUILD.bazel", + sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libzml_metal"], + root_module_direct_dev_deps = [], + ) + +metal_packages = module_extension( + implementation = _metal_impl, +) diff --git a/platforms/metal/metal.h b/platforms/metal/metal.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/metal/metal.h @@ -0,0 +1 @@ + #include diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig new file mode 100644 index 00000000..ddde2a30 --- /dev/null +++ b/platforms/metal/metal.zig @@ -0,0 +1,53 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/metal"); + +pub fn isEnabled() bool { + return platforms_options.metal_enabled; +} + +fn setMetalToolchainEnv(r: anytype) !void { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = try r.rlocation("zml/platforms/metal/metal_toolchain", &buf) orelse + return error.ToolchainNotInRunfiles; + + var bindir_z: [std.Io.Dir.max_path_bytes]u8 = undefined; + _ = c.setenv("METAL_TOOLCHAIN", try stdx.Io.Dir.path.bufJoinZ(&bindir_z, &.{ root, "bin" }), 0); +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = io; + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_metal/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for Metal runtime", .{}); + return error.FileNotFound; + }; + + setMetalToolchainEnv(r) catch |err| { + log.err("Bundled Metal toolchain not found in runfiles ({any})", .{err}); + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_metal.dylib" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/metal/metal_toolchain.BUILD.bazel b/platforms/metal/metal_toolchain.BUILD.bazel new file mode 100644 index 00000000..ca70fff0 --- /dev/null +++ b/platforms/metal/metal_toolchain.BUILD.bazel @@ -0,0 +1,382 @@ +package(default_visibility = ["//visibility:public"]) + +# Static BUILD file for the 17B54 MetalToolchain asset. + +filegroup( + name = "toolchain_runtime", + srcs = glob(["Metal.xctoolchain/**"]), +) + +exports_files([ + "Metal.xctoolchain/ToolchainInfo.plist", + "Metal.xctoolchain/usr/bin/air-ar", + "Metal.xctoolchain/usr/bin/air-arch", + "Metal.xctoolchain/usr/bin/air-as", + "Metal.xctoolchain/usr/bin/air-binary-perf", + "Metal.xctoolchain/usr/bin/air-config", + "Metal.xctoolchain/usr/bin/air-dsymutil", + "Metal.xctoolchain/usr/bin/air-libtool", + "Metal.xctoolchain/usr/bin/air-link", + "Metal.xctoolchain/usr/bin/air-lipo", + "Metal.xctoolchain/usr/bin/air-lld", + "Metal.xctoolchain/usr/bin/air-nm", + "Metal.xctoolchain/usr/bin/air-nt", + "Metal.xctoolchain/usr/bin/air-objcopy", + "Metal.xctoolchain/usr/bin/air-objdump", + "Metal.xctoolchain/usr/bin/air-opt", + "Metal.xctoolchain/usr/bin/air-pack", + "Metal.xctoolchain/usr/bin/air-package", + "Metal.xctoolchain/usr/bin/air-ranlib", + "Metal.xctoolchain/usr/bin/air-readobj", + "Metal.xctoolchain/usr/bin/air-size", + "Metal.xctoolchain/usr/bin/air-strip", + "Metal.xctoolchain/usr/bin/air-tt", + "Metal.xctoolchain/usr/bin/air-vtool", + "Metal.xctoolchain/usr/bin/amdgpu-nt", + "Metal.xctoolchain/usr/bin/applegpu-nt", + "Metal.xctoolchain/usr/bin/intelgpu-nt", + "Metal.xctoolchain/usr/bin/metal", + "Metal.xctoolchain/usr/bin/metal-ar", + "Metal.xctoolchain/usr/bin/metal-arch", + "Metal.xctoolchain/usr/bin/metal-as", + "Metal.xctoolchain/usr/bin/metal-binary-perf", + "Metal.xctoolchain/usr/bin/metal-config", + "Metal.xctoolchain/usr/bin/metal-dsymutil", + "Metal.xctoolchain/usr/bin/metal-libtool", + "Metal.xctoolchain/usr/bin/metal-link", + "Metal.xctoolchain/usr/bin/metal-lipo", + "Metal.xctoolchain/usr/bin/metal-lld", + "Metal.xctoolchain/usr/bin/metal-nm", + "Metal.xctoolchain/usr/bin/metal-nt", + "Metal.xctoolchain/usr/bin/metal-objcopy", + "Metal.xctoolchain/usr/bin/metal-objdump", + "Metal.xctoolchain/usr/bin/metal-opt", + "Metal.xctoolchain/usr/bin/metal-pack", + "Metal.xctoolchain/usr/bin/metal-package", + "Metal.xctoolchain/usr/bin/metal-ranlib", + "Metal.xctoolchain/usr/bin/metal-readobj", + "Metal.xctoolchain/usr/bin/metal-size", + "Metal.xctoolchain/usr/bin/metal-source", + "Metal.xctoolchain/usr/bin/metal-strip", + "Metal.xctoolchain/usr/bin/metal-tt", + "Metal.xctoolchain/usr/bin/metal-vtool", + "Metal.xctoolchain/usr/bin/metallib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_ios.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iosmac.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_osx.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xros.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xrossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLIGCCompilerPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLPasses.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu23-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu24-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpuG9G12-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libigc.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/metalfe/libmetal.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.gpuencode.split", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.igc64.split", + "Metal.xctoolchain/usr/metal/32023/lib/wrapper-nt.dylib", +]) + +alias(name = "toolchain_info", actual = "Metal.xctoolchain/ToolchainInfo.plist") + +alias( + name = "air-ar", + actual = "Metal.xctoolchain/usr/bin/air-ar", +) + +alias( + name = "air-arch", + actual = "Metal.xctoolchain/usr/bin/air-arch", +) + +alias( + name = "air-as", + actual = "Metal.xctoolchain/usr/bin/air-as", +) + +alias( + name = "air-binary-perf", + actual = "Metal.xctoolchain/usr/bin/air-binary-perf", +) + +alias( + name = "air-config", + actual = "Metal.xctoolchain/usr/bin/air-config", +) + +alias( + name = "air-dsymutil", + actual = "Metal.xctoolchain/usr/bin/air-dsymutil", +) + +alias( + name = "air-libtool", + actual = "Metal.xctoolchain/usr/bin/air-libtool", +) + +alias( + name = "air-link", + actual = "Metal.xctoolchain/usr/bin/air-link", +) + +alias( + name = "air-lipo", + actual = "Metal.xctoolchain/usr/bin/air-lipo", +) + +alias( + name = "air-lld", + actual = "Metal.xctoolchain/usr/bin/air-lld", +) + +alias( + name = "air-nm", + actual = "Metal.xctoolchain/usr/bin/air-nm", +) + +alias( + name = "air-nt", + actual = "Metal.xctoolchain/usr/bin/air-nt", +) + +alias( + name = "air-objcopy", + actual = "Metal.xctoolchain/usr/bin/air-objcopy", +) + +alias( + name = "air-objdump", + actual = "Metal.xctoolchain/usr/bin/air-objdump", +) + +alias( + name = "air-opt", + actual = "Metal.xctoolchain/usr/bin/air-opt", +) + +alias( + name = "air-pack", + actual = "Metal.xctoolchain/usr/bin/air-pack", +) + +alias( + name = "air-package", + actual = "Metal.xctoolchain/usr/bin/air-package", +) + +alias( + name = "air-ranlib", + actual = "Metal.xctoolchain/usr/bin/air-ranlib", +) + +alias( + name = "air-readobj", + actual = "Metal.xctoolchain/usr/bin/air-readobj", +) + +alias( + name = "air-size", + actual = "Metal.xctoolchain/usr/bin/air-size", +) + +alias( + name = "air-strip", + actual = "Metal.xctoolchain/usr/bin/air-strip", +) + +alias( + name = "air-tt", + actual = "Metal.xctoolchain/usr/bin/air-tt", +) + +alias( + name = "air-vtool", + actual = "Metal.xctoolchain/usr/bin/air-vtool", +) + +alias( + name = "amdgpu-nt", + actual = "Metal.xctoolchain/usr/bin/amdgpu-nt", +) + +alias( + name = "applegpu-nt", + actual = "Metal.xctoolchain/usr/bin/applegpu-nt", +) + +alias( + name = "intelgpu-nt", + actual = "Metal.xctoolchain/usr/bin/intelgpu-nt", +) + +alias( + name = "metal", + actual = "Metal.xctoolchain/usr/bin/metal", +) + +alias( + name = "metal-ar", + actual = "Metal.xctoolchain/usr/bin/metal-ar", +) + +alias( + name = "metal-arch", + actual = "Metal.xctoolchain/usr/bin/metal-arch", +) + +alias( + name = "metal-as", + actual = "Metal.xctoolchain/usr/bin/metal-as", +) + +alias( + name = "metal-binary-perf", + actual = "Metal.xctoolchain/usr/bin/metal-binary-perf", +) + +alias( + name = "metal-config", + actual = "Metal.xctoolchain/usr/bin/metal-config", +) + +alias( + name = "metal-dsymutil", + actual = "Metal.xctoolchain/usr/bin/metal-dsymutil", +) + +alias( + name = "metal-libtool", + actual = "Metal.xctoolchain/usr/bin/metal-libtool", +) + +alias( + name = "metal-link", + actual = "Metal.xctoolchain/usr/bin/metal-link", +) + +alias( + name = "metal-lipo", + actual = "Metal.xctoolchain/usr/bin/metal-lipo", +) + +alias( + name = "metal-lld", + actual = "Metal.xctoolchain/usr/bin/metal-lld", +) + +alias( + name = "metal-nm", + actual = "Metal.xctoolchain/usr/bin/metal-nm", +) + +alias( + name = "metal-nt", + actual = "Metal.xctoolchain/usr/bin/metal-nt", +) + +alias( + name = "metal-objcopy", + actual = "Metal.xctoolchain/usr/bin/metal-objcopy", +) + +alias( + name = "metal-objdump", + actual = "Metal.xctoolchain/usr/bin/metal-objdump", +) + +alias( + name = "metal-opt", + actual = "Metal.xctoolchain/usr/bin/metal-opt", +) + +alias( + name = "metal-pack", + actual = "Metal.xctoolchain/usr/bin/metal-pack", +) + +alias( + name = "metal-package", + actual = "Metal.xctoolchain/usr/bin/metal-package", +) + +alias( + name = "metal-ranlib", + actual = "Metal.xctoolchain/usr/bin/metal-ranlib", +) + +alias( + name = "metal-readobj", + actual = "Metal.xctoolchain/usr/bin/metal-readobj", +) + +alias( + name = "metal-size", + actual = "Metal.xctoolchain/usr/bin/metal-size", +) + +alias( + name = "metal-source", + actual = "Metal.xctoolchain/usr/bin/metal-source", +) + +alias( + name = "metal-strip", + actual = "Metal.xctoolchain/usr/bin/metal-strip", +) + +alias( + name = "metal-tt", + actual = "Metal.xctoolchain/usr/bin/metal-tt", +) + +alias( + name = "metal-vtool", + actual = "Metal.xctoolchain/usr/bin/metal-vtool", +) + +alias( + name = "metallib", + actual = "Metal.xctoolchain/usr/bin/metallib", +) + +filegroup( + name = "metal_headers", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/include/**/*.h"]), +) + +filegroup( + name = "tracepoint_runtime_metallibs", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/lib/darwin/*.metallib"]), +) + +filegroup( + name = "metal_compile_tools", + srcs = [ + "Metal.xctoolchain/usr/metal/32023/bin/metal", + "Metal.xctoolchain/usr/metal/32023/bin/air-opt", + "Metal.xctoolchain/usr/metal/32023/bin/air-as", + "Metal.xctoolchain/usr/metal/32023/bin/metallib", + ], +) + +filegroup( + name = "metal_compile_runtime", + srcs = glob([ + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/**", + ]), +) diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index e095d025..ad537191 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -1,21 +1,32 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") +load("@zml//bazel:cc_import.bzl", "cc_import") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") +exports_files([ + "neuronx_cc_shim.py", + "nki_kernel_compiler.py", +]) + # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. # -# Additionally, it provides a way to load implicit transitive dependencies -# of neuronx-cc (see add_needed of the patchelf target below). +# The proxy also initializes the Neuron runtime environment and exposes the +# sandboxed compiler binaries to libneuronxla 3.0. zig_shared_library( - name = "libpjrt_neuron", - main = "libpjrt_neuron.zig", - visibility = ["@libpjrt_neuron//:__subpackages__"], + name = "libzml_neuron", + srcs = ["neuron.zig"], + main = "libzml_neuron.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = [ - ":libpython", + ":libnrt_headers", + ":neuron_nrt", + "//bazel", + "//pjrt", + "//platforms:options", "//stdx", "@rules_zig//zig/runfiles", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", @@ -26,67 +37,109 @@ zig_binary( name = "neuronx-cc", data = ["@neuron_py_deps//neuronx_cc"], linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], - main = "neuronx-cc.zig", + main = "neuronxcc_compiler_launcher.zig", tags = ["manual"], - deps = [":libpython"], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + ":libpython", + ":python_launcher", + ], ) runfiles_to_default( name = "neuronx-cc_files", - visibility = ["@libpjrt_neuron//:__subpackages__"], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], deps = [":neuronx-cc"], ) +runfiles_to_default( + name = "nki-cc_files", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], + deps = [":nki-cc"], +) + cc_library( name = "libpython", hdrs = ["libpython.h"], + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], deps = [ "@rules_python//python/cc:current_py_cc_headers", "@rules_python//python/cc:current_py_cc_libs", ], ) -cc_library( - name = "empty", - defines = ["ZML_RUNTIME_NEURON_DISABLED"], -) - -cc_library( - name = "zmlxneuron", - defines = ["ZML_RUNTIME_NEURON"], -) - cc_library( name = "libnrt_headers", hdrs = ["nrt.h"], - deps = ["@libpjrt_neuron//:libnrt_headers"], -) - -upb_c_proto_library( - name = "xla_data_upb", - deps = ["@xla//xla:xla_data_proto"], -) - -upb_c_proto_library( - name = "hlo_proto_upb", - deps = ["@xla//xla/service:hlo_proto"], -) - -zig_shared_library( - name = "libneuronxla", - main = "libneuronxla.zig", - shared_lib_name = "libneuronxla.so", - visibility = ["@libpjrt_neuron//:__subpackages__"], - zigopts = [ - "-fno-stack-check", - "-fPIC", + tags = ["manual"], + visibility = [ + "//bin/zml-smi:__subpackages__", ], + deps = ["@libzml_neuron//:libnrt_headers"], +) + +cc_import( + name = "neuron_ncfw", + shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", + tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], +) + +cc_import( + name = "neuron_nrt", + shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", + tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + deps = [":neuron_ncfw"], +) + +zig_library( + name = "python_launcher", + import_name = "platforms/neuron/python_launcher", + main = "python_launcher.zig", + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], + deps = [":libpython"], +) + +zig_binary( + name = "nki-cc", + data = [ + "nki_kernel_compiler.py", + "@neuron_py_deps//neuronx_cc", + "@neuron_py_deps//nki", + ], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], + main = "nki_compiler_launcher.zig", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//platforms/neuron:__pkg__"], deps = [ - ":hlo_proto_upb", ":libpython", - ":xla_data_upb", + ":python_launcher", + ], +) + +zig_library( + name = "nki_kernel", + import_name = "platforms/neuron/nki_kernel", + main = "nki_kernel.zig", + tags = ["manual"], + visibility = ["//zml:__pkg__"], + deps = [ + ":neuron", + "//bazel", "//stdx", - "//upb", + "@rules_zig//zig/runfiles", ], ) @@ -101,7 +154,6 @@ uv_lock( "--upgrade", "--python-platform=x86_64-unknown-linux-gnu", ], - python_version = "3.12", tags = ["manual"], ) @@ -109,19 +161,32 @@ zig_library( name = "neuron", import_name = "platforms/neuron", main = "neuron.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:neuron.enabled": [ ":libnrt_headers", - ":libpython", - ":zmlxneuron", + ":neuron_nrt", "//bazel", "//stdx", - "@libpjrt_neuron", + "@libzml_neuron", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) + +zig_library( + name = "topology", + data = [":libzml_neuron"], + import_name = "platforms/neuron/topology", + main = "topology.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":neuron", + ], +) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig deleted file mode 100644 index 8185cf56..00000000 --- a/platforms/neuron/libneuronxla.zig +++ /dev/null @@ -1,256 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const stdx = @import("stdx"); -const upb = @import("upb"); - -const log = std.log.scoped(.@"zml/platforms/neuron/libneuronxla"); - -pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { - const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; - const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ - tmp_dir, - std.Io.Dir.path.sep_str_posix, - prefix, - (try std.Io.Clock.now(.real, io)).toNanoseconds(), - }); - try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); - return ret; -} - -const ModuleState = struct { - threaded: std.Io.Threaded, -}; - -var module_def: c.PyModuleDef = .{ - .m_base = std.mem.zeroes(c.PyModuleDef_Base), - .m_doc = "Zig bindings for libneuronxla", - .m_name = "libneuronxla", - .m_size = @sizeOf(ModuleState), - .m_methods = @constCast(&[_]c.PyMethodDef{ - .{ - .ml_name = "hook", - .ml_meth = @ptrCast(&hook), - .ml_flags = c.METH_NOARGS, - .ml_doc = "Return a greeting from Zig.", - }, - .{ - .ml_name = "neuronx_cc", - .ml_meth = @ptrCast(&neuronx_cc), - .ml_flags = c.METH_FASTCALL, - .ml_doc = "Return a greeting from Zig.", - }, - std.mem.zeroes(c.PyMethodDef), - }), - .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ - .{ .slot = c.Py_mod_exec, .value = @ptrCast(@constCast(&module_exec)) }, - std.mem.zeroes(c.PyModuleDef_Slot), - }), - .m_traverse = null, - .m_clear = null, - .m_free = null, -}; - -fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { - const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); - state.* = .{ - .threaded = .init(std.heap.c_allocator, .{}), - }; - return 0; -} - -fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { - _ = self; - _ = args; - const none = c.Py_None(); - defer c.Py_IncRef(none); - return none; -} - -pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { - var buf: [*c]u8 = undefined; - var len: c.Py_ssize_t = undefined; - _ = c.PyBytes_AsStringAndSize(object, &buf, &len); - return buf[0..@intCast(len)]; -} - -fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, io: std.Io, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { - var upb_alloc: upb.Allocator = .init(allocator); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - - const hlo_module = try upb.parse(c.xla_HloModuleProto, upb_arena, hlo_code); - - const entry = blk: { - var size: usize = undefined; - const computations = c.xla_HloModuleProto_mutable_computations(hlo_module, &size)[0..size]; - for (computations) |comp| { - if (c.xla_HloComputationProto_id(comp) == c.xla_HloModuleProto_entry_computation_id(hlo_module)) { - break :blk comp; - } - } else return error.ComputationNotFound; - }; - - const entry_instructions = blk: { - var size: usize = undefined; - break :blk c.xla_HloComputationProto_instructions(entry, &size)[0..size]; - }; - c.xla_HloComputationProto_clear_instructions(entry); - - const fused_root = blk: { - for (entry_instructions) |instruction| { - if (c.xla_HloInstructionProto_id(instruction) == c.xla_HloComputationProto_root_id(entry)) { - break :blk try upb.shallowClone(c.xla_HloInstructionProto, upb_arena, instruction); - } - } else return error.ComputationNotFound; - }; - - c.xla_HloInstructionProto_set_opcode(fused_root, upb.stringView("custom-call")); - c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); - c.xla_HloInstructionProto_set_backend_config( - fused_root, - upb.stringView( - try stdx.Io.Dir.readFileAlloc(.cwd(), io, neff_file_path, allocator, .unlimited), - ), - ); - - const parameters_len = blk: { - var size: usize = undefined; - _ = c.xla_ProgramShapeProto_parameters( - c.xla_HloComputationProto_program_shape(entry), - &size, - ); - break :blk size; - }; - - { - var operand_ids: std.ArrayList(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); - var new_instructions: std.ArrayList(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); - for (entry_instructions) |instruction| { - if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { - const id = c.xla_HloInstructionProto_id(instruction); - operand_ids.appendAssumeCapacity(id); - new_instructions.appendAssumeCapacity(instruction); - } - } - new_instructions.appendAssumeCapacity(fused_root); - } - - { - const fa = c.xla_HloInstructionProto_mutable_frontend_attributes(fused_root, upb_arena); - const map = c._xla_FrontendAttributes_map_mutable_upb_map(fa, upb_arena); - _ = c.upb_Map_Set( - map, - .{ .str_val = upb.stringView("valid_inputs") }, - .{ .str_val = blk: { - const valid_inputs_value = try allocator.alloc(u8, parameters_len * 2 - 1); - for (valid_inputs_value, 0..) |*char, i| { - char.* = if (i % 2 == 0) '1' else ','; - } - break :blk upb.stringView(valid_inputs_value); - } }, - upb_arena, - ); - } - - return try upb.serialize(hlo_module, upb_arena); -} - -fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { - const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(self))); - const io = state.threaded.io(); - - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); - defer arena.deinit(); - - const args = args_[0..@intCast(nargs_)]; - - const code = PyBytes_AsStringAndSize(args[0]); - const platform_version = PyBytes_AsStringAndSize(args[2]); - - const target = std.StaticStringMap([]const u8).initComptime(.{ - .{ "1.0", "inf1" }, - .{ "2.0", "trn1" }, - .{ "3.0", "trn2" }, - }).get(platform_version) orelse { - log.err("Unknown platform version: {s}\n", .{platform_version}); - return error.UnknownPlatformVersion; - }; - - var tmp_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const tmp_dir = try makeTempDir(io, &tmp_dir_buf, "zml-neuronxcc-"); - defer std.Io.Dir.deleteTree(.cwd(), io, tmp_dir) catch |err| { - log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); - }; - - const code_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); - try std.Io.Dir.writeFile(.cwd(), io, .{ - .data = code, - .sub_path = code_file, - .flags = .{ .truncate = true }, - }); - - const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); - - var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - var child: std.process.Child = .init(&.{ - try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ - stdx.process.selfSharedObjectDirPath(), - "..", - "bin", - "neuronx-cc", - }), - "compile", - "--framework=XLA", - "--target", - target, - "--verbose=user", - "--enable-internal-neff-wrapper", - "--output", - neff_file, - "--optlevel=1", - // generic is the default, but it fails on transformers, force it - "--model-type=transformer", - // disable it, we do our own - "--auto-cast=none", - "--enable-fast-loading-neuron-binaries", - code_file, - }, arena.allocator()); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Inherit; - child.stderr_behavior = .Inherit; - child.cwd = tmp_dir; - _ = try child.spawnAndWait(io); - - std.debug.print(">>>> {s}\n", .{tmp_dir}); - - const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { - log.err("Error wrapping NEFF as custom call: {}\n", .{err}); - return err; - }; - - return c.PyTuple_Pack( - 2, - c.PyLong_FromLongLong(0), - c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), - ); -} - -fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { - return neuronx_cc_(self, args_, nargs_) catch |err| { - log.err("Error in neuronx_cc: {}\n", .{err}); - - const none = c.Py_None(); - c.Py_IncRef(none); - const tuple = c.PyTuple_New(2) orelse { - c.Py_DecRef(none); - return null; - }; - _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); - _ = c.PyTuple_SetItem(tuple, 1, none); - return tuple; - }; -} - -pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { - return c.PyModuleDef_Init(&module_def); -} diff --git a/platforms/neuron/libpjrt_neuron.BUILD.bazel b/platforms/neuron/libzml_neuron.BUILD.bazel similarity index 56% rename from platforms/neuron/libpjrt_neuron.BUILD.bazel rename to platforms/neuron/libzml_neuron.BUILD.bazel index df831ad4..8d8a7119 100644 --- a/platforms/neuron/libpjrt_neuron.BUILD.bazel +++ b/platforms/neuron/libzml_neuron.BUILD.bazel @@ -1,5 +1,6 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:select_file.bzl", "select_file") +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") @@ -16,38 +17,86 @@ select_file( patchelf( name = "libneuronpjrt_so", - set_rpath = "$ORIGIN", src = ":libneuronpjrt_so_orig", + set_rpath = "$ORIGIN", soname = "libneuronpjrt.so", ) patchelf( - name = "libpjrt_neuron_so", + name = "libzml_neuron_so", + src = "@zml//platforms/neuron:libzml_neuron", set_rpath = "$ORIGIN", - src = "@zml//platforms/neuron:libpjrt_neuron", - soname = "libpjrt_neuron.so", + soname = "libzml_neuron.so", ) +patchelf( + name = "neuron_explorer", + src = "@aws-neuronx-tools//:neuron-explorer", + add_needed = ["libzml_neuron.so"], + set_rpath = "$ORIGIN/../lib", +) + +# This is a compiler-enabled sandbox. Python stays packaged because AWS exposes +# neuronx-cc and NKI compilation through Python entrypoints; ZML wraps those as +# native ELF launchers in bin/ so the image does not need shell scripts or host +# Python. Keep AWS compiler/runtime payloads intact and only trim Python +# files that are not part of Neuron compilation. copy_to_directory( name = "sandbox", srcs = [ ":libneuronpjrt_so", - ":libpjrt_neuron_so", - "@zml//platforms/neuron:neuronx-cc_files", - "@zml//platforms/neuron:libneuronxla", - "@rules_python//python:current_py_toolchain", - "@aws-neuronx-runtime-lib//:libnrt.patchelf", - "@aws-neuronx-runtime-lib//:libncfw.patchelf", - "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + ":libneuronxla_files", + ":libzml_neuron_so", + ":neuron_explorer", "@aws-neuronx-collectives//:libnccom", - "@zlib1g", + "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-runtime-lib//:libnrt.patchelf", + "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + "@libgcc_s1", "@libgomp1", + "@libstdcpp6", + "@rules_python//python:current_py_toolchain", + "@zlib1g", + "@zml//platforms/neuron:neuronx-cc_files", + "@zml//platforms/neuron:neuronx_cc_shim.py", + "@zml//platforms/neuron:nki-cc_files", + "@zml//platforms/neuron:nki_kernel_compiler.py", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/__pycache__/**", + "**/*.pyc", + "**/tests/**", + "lib/libtcl*.so*", + "lib/libtk*.so*", + "lib/python*/idlelib/**", + "lib/python*/tkinter/**", + "lib/python*/turtledemo/**", + "lib/python*/ensurepip/**", + # "**/include/**", + "**/_solib_*/**", + ], + hardlink = "on", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "platforms/neuron/neuronx-cc", + "platforms/neuron/neuronx_cc_shim.py", + "platforms/neuron/nki-cc", + "platforms/neuron/nki_kernel_compiler.py", + "neuron_explorer/**", + "**/lib**", + "lib/python*/**", + "site-packages/**", ], replace_prefixes = { "platforms/neuron/neuronx-cc": "bin/neuronx-cc", - "platforms/neuron/libneuronxla.so": "site-packages/libneuronxla.so", + "platforms/neuron/neuronx_cc_shim.py": "bin/python-shims/neuronx-cc", + "platforms/neuron/nki-cc": "bin/nki-cc", + "platforms/neuron/nki_kernel_compiler.py": "site-packages/nki_kernel_compiler.py", + "neuron_explorer": "bin", "libneuronpjrt_so": "lib", - "libpjrt_neuron_so": "lib", + "libzml_neuron_so": "lib", "libnrt.patchelf": "lib", "libncfw.patchelf": "lib", "libnrtucode_extisa": "lib", @@ -55,28 +104,15 @@ copy_to_directory( "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, - exclude_srcs_patterns = [ - "**/tests/**", - # "**/include/**", - "**/_solib_*/**", + visibility = [ + "@zml//tools/neuron:__pkg__", + "@zml//zml:__pkg__", + "@zml//zml/kernels/flash_attention/nki:__pkg__", ], - include_srcs_patterns = [ - "platforms/neuron/neuronx-cc", - "**/lib**", - "lib/python*/**", - "site-packages/**", - ], - allow_overwrites = True, - add_directory_to_runfiles = False, - include_external_repositories = ["**"], - hardlink = "on", ) cc_library( name = "libnrt_headers", - deps = [ - "@aws-neuronx-runtime-lib//:libnrt_headers", - ], linkopts = [ # Defer function call resolution until the function is called # (lazy loading) rather than at load time. @@ -88,10 +124,13 @@ cc_library( "-Wl,-z,lazy", ], visibility = ["@zml//platforms/neuron:__subpackages__"], + deps = [ + "@aws-neuronx-runtime-lib//:libnrt_headers", + ], ) cc_library( - name = "libpjrt_neuron", + name = "libzml_neuron", data = [":sandbox"], visibility = ["@zml//platforms/neuron:__subpackages__"], ) diff --git a/platforms/neuron/libpjrt_neuron.zig b/platforms/neuron/libzml_neuron.zig similarity index 61% rename from platforms/neuron/libpjrt_neuron.zig rename to platforms/neuron/libzml_neuron.zig index 1d25cde6..404d8cc0 100644 --- a/platforms/neuron/libpjrt_neuron.zig +++ b/platforms/neuron/libzml_neuron.zig @@ -2,14 +2,16 @@ const std = @import("std"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); +const neuron = @import("neuron.zig"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); fn findFreeTcpPort(io: std.Io) !u16 { + const addr: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }; var server = try std.Io.net.IpAddress.listen( - .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }, + &addr, io, .{}, ); @@ -37,11 +39,14 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu } extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; -fn setupNeuronEnv(io: std.Io) !void { - var buf: [256]u8 = undefined; + +fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { + const scratchpad_page_size = "1024"; + + var root_comm_buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), + try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), 1, ); _ = setenv( @@ -52,80 +57,51 @@ fn setupNeuronEnv(io: std.Io) !void { }), 1, ); - _ = setenv( - "NEURON_RT_STOCHASTIC_ROUNDING_EN", - "1", - 1, + _ = setenv("NEURON_RT_LOG_LEVEL", "error", 0); + _ = setenv("NEURON_RT_DISABLE_EXECUTION_BARRIER", "1", 0); + _ = setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); + _ = setenv("NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS", "3", 0); + _ = setenv("NEURON_SCRATCHPAD_PAGE_SIZE", scratchpad_page_size, 0); + + var bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const bin_path = try stdx.Io.Dir.path.bufJoin(&bin_path_buf, &.{ sandbox_path, "bin" }); + + const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; + var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; + const new_path = if (old_path.len == 0) + try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) + else + try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); + if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; + + if (std.c.getenv("NEURON_CC_FLAGS") != null) return; + + const log_level = if (std.c.getenv("NEURON_RT_LOG_LEVEL")) |level| std.mem.span(level) else "error"; + + var flags_buf: [512:0]u8 = undefined; + const instance = try neuron.instance(); + const target = @tagName(instance.compilerTarget()); + const flags = try std.mem.printSentinel( + &flags_buf, + "--target={s} " ++ + "--optlevel=1 " ++ + "--model-type=transformer " ++ + "--enable-fast-loading-neuron-binaries " ++ + "--enable-fast-context-switch " ++ + "--hbm-scratchpad-page-size={s} " ++ + "--verbose {s} " ++ + "--logfile-verbose {s} " ++ + "--logfile=./log-neuron-cc.txt", + .{ + target, + scratchpad_page_size, + log_level, + log_level, + }, + 0, ); -} -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { - if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); - path_with_null[len] = 0; - return path_with_null; -} - -// redefine it because for some reason PyThreadState can't be sized -extern fn PyEval_SaveThread() *anyopaque; - -fn setupPythonEnv(sandbox_path: []const u8) !void { - const Static = struct { - var py_config: c.PyConfig = undefined; - }; - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - c.PyConfig_InitIsolatedConfig(&Static.py_config); - - Static.py_config.module_search_paths_set = 1; - Static.py_config.optimization_level = 2; - Static.py_config.write_bytecode = 0; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - sandbox_path, - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&Static.py_config, &Static.py_config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try stdx.Io.Dir.path.bufJoin(&buf, &.{ - sandbox_path, - "site-packages", - }); - pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&Static.py_config)); - - // release the GIL - _ = PyEval_SaveThread(); + _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); } // Duplicates a PJRT Api object while being careful about struct size differences @@ -155,8 +131,7 @@ fn getPjrtApi() !*c.PJRT_Api { "..", }); - try setupNeuronEnv(io); - try setupPythonEnv(sandbox_path); + try setupNeuronEnv(io, sandbox_path); Static.inner = blk: { const GetPjrtApi_inner = GetPjrtApi_blk: { diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 409904ca..8b947c04 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -1,4 +1,4 @@ -load("@python_versions//3.11:defs.bzl", _py_binary = "py_binary") +load("@python_versions//3.12:defs.bzl", _py_binary = "py_binary") load("@rules_python//python:defs.bzl", "PyInfo") load("@with_cfg.bzl", "with_cfg") load("//bazel:http_deb_archive.bzl", "http_deb_archive") @@ -6,15 +6,31 @@ load("//bazel:simple_repository.bzl", "simple_repository") load("//platforms:packages.bzl", "packages") BASE_URL = "https://apt.repos.neuron.amazonaws.com" -STRIP_PREFIX = "opt/aws/neuron" +STRIP_PREFIX = "./opt/aws/neuron" _BUILD_FILE_PRELUDE = """\ package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") """ _UBUNTU_PACKAGES = { - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), - "libgomp1": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), + "libgcc_s1": { + "package": "libgcc-s1", + "build": packages.filegroup(name = "libgcc_s1", srcs = ["lib/x86_64-linux-gnu/libgcc_s.so.1"]), + }, + "libgomp1": { + "package": "libgomp1", + "build": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), + }, + "libstdcpp6": { + "package": "libstdc++6", + "build": packages.filegroup(name = "libstdcpp6", srcs = ["usr/lib/x86_64-linux-gnu/libstdc++.so.6"]), + }, + "zlib1g": { + "package": "zlib1g", + "build": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + }, } _NEURON_PACKAGES = { @@ -57,6 +73,12 @@ _NEURON_PACKAGES = { srcs = ["lib/libnccom.so.2"], ), ]), + "aws-neuronx-tools": "\n".join([ + packages.filegroup( + name = "neuron-explorer", + srcs = ["bin/neuron-explorer"], + ), + ]), } def _neuron_impl(mctx): @@ -65,21 +87,21 @@ def _neuron_impl(mctx): ]) simple_repository( - name = "libpjrt_neuron", - build_file = ":libpjrt_neuron.BUILD.bazel", + name = "libzml_neuron", + build_file = ":libzml_neuron.BUILD.bazel", ) - for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + for repo_name, package in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[package["package"]]["amd64"] http_deb_archive( - name = pkg_name, + name = repo_name, urls = pkg["urls"], sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_PRELUDE + build_file_content, + build_file_content = _BUILD_FILE_PRELUDE + package["build"], ) for pkg_name, build_file_content in _NEURON_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -90,7 +112,15 @@ def _neuron_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", + root_module_direct_deps = [ + "aws-neuronx-collectives", + "aws-neuronx-runtime-lib", + "aws-neuronx-tools", + "libgcc_s1", + "libgomp1", + "libstdcpp6", + "libzml_neuron", + ], root_module_direct_dev_deps = [], ) @@ -101,4 +131,4 @@ neuron_packages = module_extension( py_binary_with_script, _py_binary_internal = with_cfg( kind = _py_binary, extra_providers = [PyInfo], -).set(Label("@rules_python//python/config_settings:bootstrap_impl"), "script").build() +).build() diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index 8751f5ac..e0365fe9 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -5,12 +5,98 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); +pub const CompilerTarget = enum { + trn1, + inf2, + trn1n, + trn2, + trn2n, + trn3, +}; + +pub const Family = enum(u32) { + unknown = 0, + inf1 = 1, + trn1 = 2, + trn1n = 3, + inf2 = 4, + trn2 = 5, + trn2n = 6, + inf2e = 7, + trn2p = 8, + trn2u = 9, + trn2e = 10, + trn2eu = 11, + trn2ac = 12, + trn2uac = 13, + trn3 = 14, + trn3pds98 = 15, + + pub fn compilerTarget(self: Family) CompilerTarget { + return switch (self) { + .unknown, .inf1 => unreachable, + .trn1 => .trn1, + .trn1n => .trn1n, + .inf2, .inf2e => .inf2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => .trn2, + .trn3, .trn3pds98 => .trn3, + }; + } +}; + +pub const Size = enum(u32) { + xl1 = 0, + xl2 = 1, + xl4 = 2, + xl6 = 3, + xl8 = 4, + xl24 = 5, + xl32 = 6, + xl48 = 7, + xl3 = 8, + unknown = 9, +}; + +pub const Instance = struct { + family: Family, + size: Size, + + pub fn compilerTarget(self: Instance) CompilerTarget { + return self.family.compilerTarget(); + } + + pub fn coresPerChip(self: Instance) !usize { + return switch (self.family) { + .inf1 => 4, + .trn1, .trn1n, .inf2, .inf2e => 2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3, .trn3pds98 => 8, + .unknown => unreachable, + }; + } +}; + +pub fn instance() !Instance { + var info = std.mem.zeroes(c.struct_nrt_instance_info); + info.family = @backingInt(Family.unknown); + info.size = @backingInt(Size.unknown); + + if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { + return error.Unavailable; + } + + return .{ + .family = @fromBackingInt(@intCast(info.family)), + .size = @fromBackingInt(@intCast(info.size)), + }; +} + pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_NEURON"); + return platforms_options.neuron_enabled; } fn hasNeuronDevice(io: std.Io) bool { @@ -34,7 +120,7 @@ fn isRunningOnEC2(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } @@ -48,17 +134,17 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const r = try bazel.runfiles(io, bazel_builtin.current_repository); + const r = try bazel.runfiles(bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse { log.err("Failed to find sandbox path for NEURON runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_neuron.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig deleted file mode 100644 index e77c4858..00000000 --- a/platforms/neuron/neuronx-cc.zig +++ /dev/null @@ -1,99 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const runfiles = @import("runfiles"); - -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { - if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); - path_with_null[len] = 0; - return path_with_null; -} - -pub fn main() !void { - const allocator = std.heap.c_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - const io = threaded.io(); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - var config: c.PyConfig = undefined; - c.PyConfig_InitIsolatedConfig(&config); - defer c.PyConfig_Clear(&config); - - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - - _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); - - var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = blk: { - const n = try std.process.executableDirPath(io, &self_exe_dir_buf); - break :blk self_exe_dir_buf[0..n]; - }; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }); - pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "site-packages", - }), - }); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); - } - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const neuronxcc_main = blk: { - const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); - std.debug.print(">>> MODULE: {any}\n", .{module}); - defer c.Py_DecRef(module); - break :blk c.PyObject_GetAttrString(module, "main"); - }; - defer c.Py_DecRef(neuronxcc_main); - - const result = c.PyObject_CallNoArgs(neuronxcc_main); - defer c.Py_DecRef(result); -} diff --git a/platforms/neuron/neuronx_cc_shim.py b/platforms/neuron/neuronx_cc_shim.py new file mode 100755 index 00000000..deff0b5d --- /dev/null +++ b/platforms/neuron/neuronx_cc_shim.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Python-compatible shim for harness NKI standalone compilation.""" + +import os +import sys + + +def main() -> None: + native = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "neuronx-cc" + ) + os.execv(native, [native, *sys.argv[1:]]) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/neuronxcc_compiler_launcher.zig b/platforms/neuron/neuronxcc_compiler_launcher.zig new file mode 100644 index 00000000..066923e2 --- /dev/null +++ b/platforms/neuron/neuronxcc_compiler_launcher.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + try python_launcher.runModuleEntrypoint(init, .{ + .module_name = "neuronxcc.driver.CommandDriver", + }); +} diff --git a/platforms/neuron/nki_compiler_launcher.zig b/platforms/neuron/nki_compiler_launcher.zig new file mode 100644 index 00000000..f8a6b884 --- /dev/null +++ b/platforms/neuron/nki_compiler_launcher.zig @@ -0,0 +1,12 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + _ = std; + try python_launcher.runScriptMain(init, &.{ + "..", + "site-packages", + "nki_kernel_compiler.py", + }); +} diff --git a/platforms/neuron/nki_kernel.zig b/platforms/neuron/nki_kernel.zig new file mode 100644 index 00000000..4d897be8 --- /dev/null +++ b/platforms/neuron/nki_kernel.zig @@ -0,0 +1,160 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const stdx = @import("stdx"); + +const neuron = @import("platforms/neuron"); + +const log = std.log.scoped(.@"zml/platforms/neuron/nki_kernel"); + +pub const TensorSignature = struct { + dtype: []const u8, + dims: []const i64, +}; + +pub const Kernel = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerRequest = struct { + entrypoint: []const u8, + source: []const u8, + target: []const u8, + tool_bin_dir: []const u8, + neuronx_cc_args: []const []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerResponse = struct { + backend_config_b64: []const u8, +}; + +pub fn compilerTargetFromInstance() neuron.CompilerTarget { + const instance = neuron.instance() catch |err| { + std.debug.panic("failed to query Neuron instance for NKI compiler target: {}", .{err}); + }; + return instance.compilerTarget(); +} + +pub fn compileNkiKernel(allocator: std.mem.Allocator, io: std.Io, kernel: Kernel) ![]const u8 { + const compiler_flags = std.mem.span(std.c.getenv("NEURON_CC_FLAGS") orelse return error.MissingNeuronCompilerFlags); + var compiler_args: std.ArrayList([]const u8) = .empty; + defer compiler_args.deinit(allocator); + var compiler_flags_iter = std.mem.tokenizeScalar(u8, compiler_flags, ' '); + while (compiler_flags_iter.next()) |arg| { + try compiler_args.append(allocator, arg); + } + + const tmp_root_path = if (std.c.getenv("TMPDIR")) |tmpdir| std.mem.span(tmpdir) else "/tmp"; + var tmp_root = try std.Io.Dir.openDir(.cwd(), io, tmp_root_path, .{}); + defer tmp_root.close(io); + + var dir_name_buf: [128]u8 = undefined; + const dir_name = try std.fmt.bufPrint(&dir_name_buf, "zml-nki-{d}", .{std.Io.Timestamp.now(io, .real).nanoseconds}); + + var tmp_dir = try tmp_root.createDirPathOpen(io, dir_name, .{ .permissions = .fromMode(0o700) }); + defer tmp_dir.close(io); + + var cwd_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const cwd = try tmp_dir.realPath(io, &cwd_buf); + + const request_file = try tmp_dir.createFile(io, "kernel.request.json", .{ .read = true }); + defer request_file.close(io); + + const result_file = try tmp_dir.createFile(io, "kernel.result.json", .{ .read = true }); + defer result_file.close(io); + + var request_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const request_file_path = try request_file.realPath(io, &request_file_buf); + + var result_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const result_file_path = try result_file.realPath(io, &result_file_buf); + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; + + var source_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const source_path = try r.rlocation(kernel.source_path, &source_path_buf) orelse return error.FileNotFound; + const source = try std.Io.Dir.cwd().readFileAlloc(io, source_path, allocator, .unlimited); + defer allocator.free(source); + + var nki_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nki_cc_path = try stdx.Io.Dir.path.bufJoin(&nki_cc_buf, &.{ sandbox_path, "bin", "nki-cc" }); + + const bin_dirpath = std.fs.path.dirname(nki_cc_path) orelse return error.InvalidSandboxPath; + + const request: CompilerRequest = .{ + .entrypoint = kernel.entrypoint, + .source = source, + .target = kernel.compiler_target, + .tool_bin_dir = bin_dirpath, + .neuronx_cc_args = compiler_args.items, + .inputs = kernel.inputs, + .outputs = kernel.outputs, + }; + + const request_json = try std.fmt.allocPrint(allocator, "{f}", .{std.json.fmt(request, .{ .emit_null_optional_fields = false })}); + defer allocator.free(request_json); + + try request_file.writePositionalAll(io, request_json, 0); + + var spawn_threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); + defer spawn_threaded.deinit(); + + const spawn_io = spawn_threaded.io(); + + var child = try std.process.spawn(spawn_io, .{ + .argv = &.{ + nki_cc_path, + "--request", + request_file_buf[0..request_file_path], + "--result", + result_file_buf[0..result_file_path], + }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + .cwd = .{ .path = cwd_buf[0..cwd] }, + }); + + const term = try child.wait(spawn_io); + switch (term) { + .exited => |code| { + if (code != 0) { + log.err("nki-cc exited with code {}", .{code}); + return error.NkiCcFailed; + } + }, + .signal => |sig| { + log.err("nki-cc terminated by signal {}", .{sig}); + return error.NkiCcFailed; + }, + else => |status| { + log.err("nki-cc terminated unexpectedly: {}", .{status}); + return error.NkiCcFailed; + }, + } + + var result_reader = result_file.reader(io, &.{}); + const data = try result_reader.interface.readAlloc(allocator, try result_file.length(io)); + defer allocator.free(data); + + const parsed_response = try std.json.parseFromSlice(CompilerResponse, allocator, data, .{ + .ignore_unknown_fields = true, + }); + defer parsed_response.deinit(); + + const backend_config = try allocator.dupe(u8, parsed_response.value.backend_config_b64); + errdefer allocator.free(backend_config); + + return backend_config; +} diff --git a/platforms/neuron/nki_kernel_compiler.py b/platforms/neuron/nki_kernel_compiler.py new file mode 100755 index 00000000..207d0c26 --- /dev/null +++ b/platforms/neuron/nki_kernel_compiler.py @@ -0,0 +1,179 @@ +"""Compile inline ZML NKI kernels into Neuron custom-native backend configs. + +The `nki_kernel.zig` helper writes one JSON request per inline kernel. +This script asks NKI to produce BIR and returns the backend_config expected by +`AwsNeuronCustomNativeKernel`. +""" + +import argparse +import inspect +import json +import re +import sys +import tempfile +import types +from dataclasses import dataclass +from pathlib import Path + +import ml_dtypes +import numpy as np + + +@dataclass(frozen=True) +class TensorSpec: + dtype: str + shape: tuple[int, ...] + + +def dtype_from_name(name: str): + mapping = { + "bool": np.bool_, + "i8": np.int8, + "i16": np.int16, + "i32": np.int32, + "i64": np.int64, + "u8": np.uint8, + "u16": np.uint16, + "u32": np.uint32, + "u64": np.uint64, + "f16": np.float16, + "f32": np.float32, + "f64": np.float64, + } + + if name == "bf16": + return ml_dtypes.bfloat16 + + try: + return mapping[name] + except KeyError as exc: + raise ValueError(f"unsupported dtype: {name}") from exc + + +def parse_tensor_specs(raw_specs: list[dict]) -> tuple[TensorSpec, ...]: + return tuple( + TensorSpec( + dtype=spec["dtype"], + shape=tuple(spec.get("shape", spec["dims"])), + ) + for spec in raw_specs + ) + + +def load_kernel(source_path: Path, source: str, entrypoint: str): + module_name = re.sub(r"\W", "_", source_path.stem) or "kernel" + if module_name[0].isdigit(): + module_name = f"kernel_{module_name}" + + module = types.ModuleType(module_name) + module.__file__ = str(source_path) + module.__dict__["__builtins__"] = __builtins__ + sys.modules[module_name] = module + exec(compile(source, str(source_path), "exec"), module.__dict__) + + try: + return module.__dict__[entrypoint] + except KeyError as exc: + raise KeyError(f"entrypoint {entrypoint!r} not found in {source_path}") from exc + + +def make_inputs(inputs: tuple[TensorSpec, ...]) -> list[np.ndarray]: + """Create zero-filled specialization inputs matching the declared ZML signature.""" + arrays: list[np.ndarray] = [] + for input_spec in inputs: + arrays.append( + np.zeros( + shape=input_spec.shape, + dtype=dtype_from_name(input_spec.dtype), + ) + ) + return arrays + + +def compile_kernel( + source: str, + entrypoint: str, + target: str, + tool_bin_dir: str, + neuronx_cc_args: tuple[str, ...], + inputs: tuple[TensorSpec, ...], + outputs: tuple[TensorSpec, ...], +) -> str: + """Run the full embedded-kernel pipeline for one custom-call.""" + tmp_dir = Path(tempfile.mkdtemp(prefix="zml-nki-", dir=".")) + + source_path = tmp_dir / "kernel.py" + artifacts_dir = tmp_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=False) + + source_path.write_text(source, encoding="utf-8") + + kernel = load_kernel(source_path, source, entrypoint) + + input_names = [ + parameter.name + for parameter in inspect.signature(kernel).parameters.values() + if parameter.kind + in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + + if len(input_names) != len(inputs): + raise ValueError( + f"{entrypoint} expects {len(input_names)} inputs, ZML declared {len(inputs)}" + ) + + from nki.compiler.driver import compile_to_bir + from nki.compiler.frontend import ParserFrontend + from nki.compiler.ncc_driver import CompilationMode, CompileOptions + + compile_opts = CompileOptions( + target=target, + artifacts_dir=str(artifacts_dir), + output_path=str(artifacts_dir / "unused.neff"), + neuronx_cc_path=str(Path(tool_bin_dir) / "neuronx-cc"), + neuronx_cc_args=neuronx_cc_args, + mode=CompilationMode.INTEGRATION, + ).disable_backend_optimizations() + + nir = compile_to_bir( + kernel, + frontend=ParserFrontend(), + inputs=dict(zip(input_names, make_inputs(inputs), strict=True)), + compile_opts=compile_opts, + output_names=[f"output{i}" for i in range(len(outputs))], + ) + + return nir.build_config().backend_config_b64.decode("utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compile a ZML NKI kernel to a custom-native backend config" + ) + parser.add_argument("--request", required=True) + parser.add_argument("--result", required=True) + args = parser.parse_args() + + request_path = Path(args.request) + result_path = Path(args.result) + + request = json.loads(request_path.read_text(encoding="utf-8")) + + backend_config_b64 = compile_kernel( + source=request["source"], + entrypoint=request["entrypoint"], + target=request["target"], + tool_bin_dir=request["tool_bin_dir"], + neuronx_cc_args=tuple(request["neuronx_cc_args"]), + inputs=parse_tensor_specs(request["inputs"]), + outputs=parse_tensor_specs(request["outputs"]), + ) + + result_path.write_text( + json.dumps({"backend_config_b64": backend_config_b64}), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/nrt.h b/platforms/neuron/nrt.h index d391b8eb..9970d0bd 100644 --- a/platforms/neuron/nrt.h +++ b/platforms/neuron/nrt.h @@ -1,2 +1,5 @@ #include #include +#include +#include +#include diff --git a/platforms/neuron/packages.lock.json b/platforms/neuron/packages.lock.json index 134d55e4..2f275bba 100755 --- a/platforms/neuron/packages.lock.json +++ b/platforms/neuron/packages.lock.json @@ -3,60 +3,24 @@ { "arch": "amd64", "dependencies": [ - { - "key": "debianutils_5.7-0.5_deb12u1_amd64", - "name": "debianutils", - "version": "5.7-0.5~deb12u1" - }, { "key": "libc6_2.36-9-p-deb12u10_amd64", "name": "libc6", "version": "2.36-9+deb12u10" }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, { "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", "name": "gcc-12-base", "version": "12.2.0-14+deb12u1" - }, - { - "key": "base-files_12.4-p-deb12u11_amd64", - "name": "base-files", - "version": "12.4+deb12u11" - }, - { - "key": "mawk_1.3.4.20200120-3.1_amd64", - "name": "mawk", - "version": "1.3.4.20200120-3.1" - }, - { - "key": "libtinfo6_6.4-4_amd64", - "name": "libtinfo6", - "version": "6.4-4" } ], - "key": "bash_5.2.15-2-p-b8_amd64", - "name": "bash", - "sha256": "23910515e6b8b4fb4a3b50549464ef2efdc149b568150e2aa45115b37b6e6578", + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/bash/bash_5.2.15-2+b8_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" ], - "version": "5.2.15-2+b8" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "debianutils_5.7-0.5_deb12u1_amd64", - "name": "debianutils", - "sha256": "55f951359670eb3236c9e2ccd5fac9ccb3db734f5a22aff21589e7a30aee48c9", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/d/debianutils/debianutils_5.7-0.5~deb12u1_amd64.deb" - ], - "version": "5.7-0.5~deb12u1" + "version": "12.2.0-14+deb12u1" }, { "arch": "amd64", @@ -69,17 +33,6 @@ ], "version": "2.36-9+deb12u10" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, { "arch": "amd64", "dependencies": [], @@ -91,39 +44,6 @@ ], "version": "12.2.0-14+deb12u1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "base-files_12.4-p-deb12u11_amd64", - "name": "base-files", - "sha256": "c0b28aa96c88ca2e2f18c44911b6cd07de9335e0de7922fba778886a50061e64", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/base-files/base-files_12.4+deb12u11_amd64.deb" - ], - "version": "12.4+deb12u11" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "mawk_1.3.4.20200120-3.1_amd64", - "name": "mawk", - "sha256": "bcbc83f391854ea9d50ce2a4101aacf330de3b8b71d81a798faadba14a157f78", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/m/mawk/mawk_1.3.4.20200120-3.1_amd64.deb" - ], - "version": "1.3.4.20200120-3.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtinfo6_6.4-4_amd64", - "name": "libtinfo6", - "sha256": "072d908f38f51090ca28ca5afa3b46b2957dc61fe35094c0b851426859a49a51", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/n/ncurses/libtinfo6_6.4-4_amd64.deb" - ], - "version": "6.4-4" - }, { "arch": "amd64", "dependencies": [ @@ -151,6 +71,33 @@ ], "version": "12.2.0-14+deb12u1" }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, { "arch": "amd64", "dependencies": [ @@ -181,13 +128,13 @@ { "arch": "amd64", "dependencies": [], - "key": "aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64", + "key": "aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64", "name": "aws-neuronx-runtime-lib", - "sha256": "60f5dc0c8731fdd48a9726493375dbb0f0d178d05d84f70716bcc36789fd9be9", + "sha256": "ed69354674b926490d917a6aeb7dea8b6300278a37958ee9a412e2cd89c41792", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64.deb" ], - "version": "2.29.40.0-f954cd7a5" + "version": "2.32.31.0-0234f5ed2" }, { "arch": "amd64", @@ -333,13 +280,13 @@ "version": "252.36-1~deb12u1" } ], - "key": "aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64", + "key": "aws-neuronx-collectives_2.32.28.0-452cba8de_amd64", "name": "aws-neuronx-collectives", - "sha256": "c4bd37dfd6a18883ea87b9e7e665ee266f3e344536d0f4e9c9d6da4aa38e7417", + "sha256": "ec737c6647b6fbd38550e11f25e27d0f1a96dbb2cd28ae5bbf3f827faba5bdc7", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.32.28.0-452cba8de_amd64.deb" ], - "version": "2.29.41.0-681fef5f5" + "version": "2.32.28.0-452cba8de" }, { "arch": "amd64", @@ -615,6 +562,17 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/s/systemd/libudev1_252.36-1~deb12u1_amd64.deb" ], "version": "252.36-1~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "aws-neuronx-tools_2.30.10.0-39feaa591_amd64", + "name": "aws-neuronx-tools", + "sha256": "b972e4db5e84a429aa237d1eb9347fcaa78c19e848049a3495efef734a0b240f", + "urls": [ + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-tools/aws-neuronx-tools_2.30.10.0-39feaa591_amd64.deb" + ], + "version": "2.30.10.0-39feaa591" } ], "version": 1 diff --git a/platforms/neuron/packages.yaml b/platforms/neuron/packages.yaml index 6c3de28a..eb399f49 100644 --- a/platforms/neuron/packages.yaml +++ b/platforms/neuron/packages.yaml @@ -18,8 +18,10 @@ archs: - "amd64" packages: - - "bash" # needed to launch neuronx-cc + - "libgcc-s1" # implicit dep of libneuronpjrt/libstdc++ - "libgomp1" # implicit dep of neuronx-cc - - "zlib1g" # implicit dep of libpython + - "libstdc++6" # implicit dep of libneuronpjrt/libnccom + - "zlib1g" # implicit dep of the sandboxed Python runtime - "aws-neuronx-runtime-lib" - "aws-neuronx-collectives" + - "aws-neuronx-tools" diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig new file mode 100644 index 00000000..075034ab --- /dev/null +++ b/platforms/neuron/python_launcher.zig @@ -0,0 +1,155 @@ +const std = @import("std"); + +const c = @import("c"); + +// Shared isolated-Python launcher used by Neuron helper binaries. The sandbox +// contains the Python stdlib, site-packages, and these tiny Zig executables; the +// helpers configure Python from their own location so they are hermetic. + +pub const ModuleEntrypoint = struct { + module_name: [:0]const u8, + attr_name: [:0]const u8 = "main", +}; + +// Run `module.attr()` inside the sandboxed Python interpreter. +pub fn runModuleEntrypoint(init: std.process.Init, entrypoint: ModuleEntrypoint) !void { + var config: c.PyConfig = undefined; + _ = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const module = c.PyImport_ImportModule(entrypoint.module_name.ptr) orelse { + std.log.err("Failed to import {s}", .{entrypoint.module_name}); + c.PyErr_Print(); + return error.FailedToImportModule; + }; + defer c.Py_DecRef(module); + + const main_fn = c.PyObject_GetAttrString(module, entrypoint.attr_name.ptr) orelse { + std.log.err("Failed to resolve {s}.{s}", .{ entrypoint.module_name, entrypoint.attr_name }); + c.PyErr_Print(); + return error.FailedToResolveEntrypoint; + }; + defer c.Py_DecRef(main_fn); + + const result = c.PyObject_CallNoArgs(main_fn); + if (result == null) { + c.PyErr_Print(); + std.process.exit(1); + } + defer c.Py_DecRef(result); +} + +// Run a Python file relative to the helper binary and call its `main()`. +pub fn runScriptMain(init: std.process.Init, relative_script_path: []const []const u8) !void { + const arena = init.arena; + + var config: c.PyConfig = undefined; + const self_exe_dir = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const script_path = try std.Io.Dir.path.join(arena.allocator(), relative_script_path); + const full_script_path = try std.Io.Dir.path.join(arena.allocator(), &.{ self_exe_dir, script_path }); + const bootstrap = try std.fmt.allocPrint(arena.allocator(), + \\import importlib.util + \\spec = importlib.util.spec_from_file_location("_zml_neuron_python_launcher", "{s}") + \\module = importlib.util.module_from_spec(spec) + \\assert spec.loader is not None + \\spec.loader.exec_module(module) + \\module.main() + , .{full_script_path}); + + if (c.PyRun_SimpleStringFlags(bootstrap.ptr, null) != 0) { + c.PyErr_Print(); + return error.CompilerExecutionFailed; + } +} + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } + path_with_null[len] = 0; + return path_with_null; +} + +fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { + const arena = init.arena; + const io = init.io; + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + c.PyConfig_InitIsolatedConfig(config); + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + const args = try init.minimal.args.toSlice(arena.allocator()); + const c_args = try arena.allocator().alloc([*:0]const u8, args.len); + for (args, 0..) |arg, i| { + c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; + } + pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }, 0); + pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + return self_exe_dir; +} diff --git a/platforms/neuron/requirements.in b/platforms/neuron/requirements.in index f574686b..51174303 100644 --- a/platforms/neuron/requirements.in +++ b/platforms/neuron/requirements.in @@ -1,4 +1,5 @@ --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html --extra-index-url=https://pip.repos.neuron.amazonaws.com -libneuronxla>=2.2 -neuronx-cc>=2.22 +libneuronxla>=3.0 +neuronx-cc>=2.25 +nki>=0.4 diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index 19c7bb90..0945aa7d 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,143 +4,159 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.42.24 \ - --hash=sha256:8ed6ad670a5a2d7f66c1b0d3362791b48392c7a08f78479f5d8ab319a4d9118f \ - --hash=sha256:c47a2f40df933e3861fc66fd8d6b87ee36d4361663a7e7ba39a87f5a78b2eae1 +boto3==1.43.14 \ + --hash=sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4 \ + --hash=sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d # via libneuronxla -botocore==1.42.24 \ - --hash=sha256:8fca9781d7c84f7ad070fceffaff7179c4aa7a5ffb27b43df9d1d957801e0a8d \ - --hash=sha256:be8d1bea64fb91eea08254a1e5fea057e4428d08e61f4e11083a02cafc1f8cc6 +botocore==1.43.14 \ + --hash=sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658 \ + --hash=sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516 # via # boto3 # libneuronxla # s3transfer -certifi==2026.1.4 \ - --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ - --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d # via requests -charset-normalizer==3.4.4 \ - --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ - --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ - --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ - --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ - --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ - --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ - --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ - --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ - --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ - --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ - --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ - --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ - --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ - --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ - --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ - --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ - --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ - --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ - --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ - --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ - --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ - --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ - --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ - --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ - --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ - --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ - --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ - --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ - --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ - --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ - --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ - --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ - --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ - --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ - --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ - --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ - --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ - --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ - --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ - --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ - --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ - --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ - --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ - --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ - --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ - --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ - --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ - --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ - --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ - --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ - --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ - --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ - --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ - --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ - --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ - --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ - --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ - --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ - --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ - --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ - --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ - --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ - --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ - --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ - --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ - --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ - --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ - --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ - --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ - --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ - --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ - --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ - --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ - --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ - --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ - --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ - --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ - --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ - --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ - --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ - --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ - --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ - --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ - --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ - --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ - --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ - --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ - --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ - --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ - --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ - --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ - --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ - --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ - --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ - --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ - --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ - --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ - --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ - --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ - --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ - --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ - --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ - --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ - --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ - --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ - --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ - --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ - --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ - --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ - --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ - --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ - --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ - --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 # via requests ec2-metadata==2.17.0 \ --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d # via neuronx-cc -idna==3.11 \ - --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ - --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 +idna==3.16 \ + --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ + --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d # via requests islpy==2023.2.5 \ --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ @@ -184,15 +200,15 @@ islpy==2023.2.5 \ --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f \ --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a # via neuronx-cc -jmespath==1.0.1 \ - --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ - --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 # via # boto3 # botocore -libneuronxla==2.2.14584.0+06ac23d1 \ - --hash=sha256:d1e594b27716bc59b937ccd8f40e7f2b74f6c309643e83dcf511b7ea392924f2 - # via -r runtimes/neuron/requirements.in +libneuronxla==3.0.2891.0+e2a4b1f5 \ + --hash=sha256:b453709b37565902acc85bcfb4011d55e99381b81d831a5559847bba71f9cec3 + # via -r platforms/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa @@ -237,131 +253,137 @@ ml-dtypes==0.5.4 \ --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 - # via neuronx-cc + # via + # neuronx-cc + # nki networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.22.12471.0+b4a00d10 \ - --hash=sha256:19fcf633f6f453b73e791b6f23d0e3f741fd967a61b7b82208bd74d80908c81a +neuronx-cc==2.25.3371.0+f524f7f8 \ + --hash=sha256:c71cb7333d5320b6ea91e13e0168eec01be5e7ddfd5515bf62aa47088ee274c2 # via - # -r runtimes/neuron/requirements.in + # -r platforms/neuron/requirements.in # libneuronxla -numpy==2.4.0 \ - --hash=sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee \ - --hash=sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3 \ - --hash=sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5 \ - --hash=sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548 \ - --hash=sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d \ - --hash=sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9 \ - --hash=sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded \ - --hash=sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93 \ - --hash=sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b \ - --hash=sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8 \ - --hash=sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037 \ - --hash=sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83 \ - --hash=sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d \ - --hash=sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e \ - --hash=sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf \ - --hash=sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df \ - --hash=sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb \ - --hash=sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce \ - --hash=sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6 \ - --hash=sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667 \ - --hash=sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d \ - --hash=sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e \ - --hash=sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e \ - --hash=sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6 \ - --hash=sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471 \ - --hash=sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7 \ - --hash=sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c \ - --hash=sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344 \ - --hash=sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c \ - --hash=sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98 \ - --hash=sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344 \ - --hash=sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e \ - --hash=sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475 \ - --hash=sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e \ - --hash=sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98 \ - --hash=sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db \ - --hash=sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51 \ - --hash=sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa \ - --hash=sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934 \ - --hash=sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6 \ - --hash=sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f \ - --hash=sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3 \ - --hash=sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53 \ - --hash=sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479 \ - --hash=sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a \ - --hash=sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba \ - --hash=sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d \ - --hash=sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c \ - --hash=sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a \ - --hash=sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95 \ - --hash=sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f \ - --hash=sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea \ - --hash=sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db \ - --hash=sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b \ - --hash=sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d \ - --hash=sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346 \ - --hash=sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4 \ - --hash=sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4 \ - --hash=sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4 \ - --hash=sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6 \ - --hash=sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681 \ - --hash=sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63 \ - --hash=sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b \ - --hash=sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac \ - --hash=sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6 \ - --hash=sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544 \ - --hash=sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63 \ - --hash=sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9 \ - --hash=sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25 \ - --hash=sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059 \ - --hash=sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c \ - --hash=sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2 +nki==0.4.0+25940409122.gd30719f9 \ + --hash=sha256:09b520140e55790ba00ebe3e7d53da1e5f221ff3d6c1823414acfebe4b794879 + # via + # -r platforms/neuron/requirements.in + # neuronx-cc +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 # via # ml-dtypes # neuronx-cc + # nki # scipy pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -protobuf==6.33.2 \ - --hash=sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f \ - --hash=sha256:2981c58f582f44b6b13173e12bb8656711189c2a70250845f264b877f00b1913 \ - --hash=sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4 \ - --hash=sha256:7109dcc38a680d033ffb8bf896727423528db9163be1b6a02d6a49606dcadbfe \ - --hash=sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c \ - --hash=sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d \ - --hash=sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872 \ - --hash=sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e \ - --hash=sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43 \ - --hash=sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4 +protobuf==7.35.0 \ + --hash=sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6 \ + --hash=sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee \ + --hash=sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda \ + --hash=sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011 \ + --hash=sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6 \ + --hash=sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0 \ + --hash=sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201 \ + --hash=sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5 # via neuronx-cc -psutil==7.2.1 \ - --hash=sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1 \ - --hash=sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266 \ - --hash=sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442 \ - --hash=sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79 \ - --hash=sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf \ - --hash=sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f \ - --hash=sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679 \ - --hash=sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8 \ - --hash=sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49 \ - --hash=sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f \ - --hash=sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129 \ - --hash=sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67 \ - --hash=sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6 \ - --hash=sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17 \ - --hash=sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42 \ - --hash=sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d \ - --hash=sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672 \ - --hash=sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a \ - --hash=sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc \ - --hash=sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3 \ - --hash=sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8 +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 # via neuronx-cc python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ @@ -371,9 +393,9 @@ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via botocore -requests==2.32.5 \ - --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ - --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed # via # ec2-metadata # requests-unixsocket @@ -381,88 +403,88 @@ requests-unixsocket==0.4.1 \ --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 # via neuronx-cc -s3transfer==0.16.0 \ - --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ - --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 +s3transfer==0.17.0 \ + --hash=sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a \ + --hash=sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20 # via boto3 -scipy==1.16.3 \ - --hash=sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2 \ - --hash=sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb \ - --hash=sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a \ - --hash=sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203 \ - --hash=sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304 \ - --hash=sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959 \ - --hash=sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a \ - --hash=sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d \ - --hash=sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe \ - --hash=sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562 \ - --hash=sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb \ - --hash=sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9 \ - --hash=sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc \ - --hash=sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d \ - --hash=sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686 \ - --hash=sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97 \ - --hash=sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353 \ - --hash=sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2 \ - --hash=sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876 \ - --hash=sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78 \ - --hash=sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc \ - --hash=sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119 \ - --hash=sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9 \ - --hash=sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f \ - --hash=sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135 \ - --hash=sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234 \ - --hash=sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1 \ - --hash=sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88 \ - --hash=sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc \ - --hash=sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6 \ - --hash=sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6 \ - --hash=sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22 \ - --hash=sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511 \ - --hash=sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079 \ - --hash=sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184 \ - --hash=sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c \ - --hash=sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40 \ - --hash=sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2 \ - --hash=sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0 \ - --hash=sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e \ - --hash=sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800 \ - --hash=sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4 \ - --hash=sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657 \ - --hash=sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005 \ - --hash=sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70 \ - --hash=sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07 \ - --hash=sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e \ - --hash=sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c \ - --hash=sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc \ - --hash=sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7 \ - --hash=sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146 \ - --hash=sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733 \ - --hash=sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c \ - --hash=sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d \ - --hash=sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6 \ - --hash=sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d \ - --hash=sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26 \ - --hash=sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8 \ - --hash=sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa \ - --hash=sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b \ - --hash=sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d +scipy==1.17.1 \ + --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ + --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ + --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ + --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ + --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ + --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ + --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ + --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ + --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ + --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ + --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ + --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ + --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ + --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ + --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ + --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ + --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ + --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ + --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ + --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ + --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ + --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ + --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ + --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ + --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ + --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ + --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ + --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ + --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ + --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ + --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ + --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ + --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ + --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ + --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ + --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ + --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ + --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ + --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ + --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ + --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ + --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ + --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ + --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ + --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ + --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 # via neuronx-cc six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -tqdm==4.67.1 \ - --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ - --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 +tqdm==4.67.3 \ + --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ + --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf # via neuronx-cc typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via neuronx-cc -urllib3==2.6.3 \ - --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ - --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 # via # botocore # requests diff --git a/platforms/neuron/topology.zig b/platforms/neuron/topology.zig new file mode 100644 index 00000000..201db15f --- /dev/null +++ b/platforms/neuron/topology.zig @@ -0,0 +1,222 @@ +const std = @import("std"); + +const neuron = @import("platforms/neuron"); + +// References: +// - `neuron.Instance` exposes the NRT instance family and size used below. +// - Neuron instance families and sizes are documented here: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/index.html +// - Inf1: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf1-arch.html +// - Inf2: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html +// - Trn1 / Trn1n: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn1-arch.html +// - Trn2 / Trn2 UltraServer: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn2-arch.html +// - Trn3: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn3-arch.html +// +// The same instance-architecture pages are also the basis for the inter-chip +// topology inference below, including 2D torus and ring-over-torus fabrics. +pub const InterChipTopology = union(enum) { + none: void, + linear: usize, + ring: usize, + torus2d: struct { + x: usize, + y: usize, + }, + torus2d_ring: struct { + x: usize, + y: usize, + z: usize, + }, + point_to_point: usize, +}; + +pub const Axis = struct { + fabric: Fabric, + size: usize, + geometry: Geometry, +}; + +pub const Fabric = enum { + core, + chip_x, + chip_y, + chip_z, +}; + +pub const Geometry = enum { + linear_ring, + closed_ring, + torus, + point_to_point, +}; + +pub const Placement = struct { + input_index: usize, + nc_id: usize, + chip: usize, + core: usize, +}; + +pub const VisibleMesh = struct { + instance: neuron.Instance, + chips: usize, + cores_per_chip: usize, + inter_chip_topology: InterChipTopology, + placements: []Placement, + axes_buffer: [4]Axis, + axes_len: usize, + + pub fn deinit(self: *VisibleMesh, allocator: std.mem.Allocator) void { + allocator.free(self.placements); + self.* = undefined; + } + + pub fn axes(self: *const VisibleMesh) []const Axis { + return self.axes_buffer[0..self.axes_len]; + } + + pub fn format(self: VisibleMesh, writer: *std.Io.Writer) !void { + try writer.print( + "VisibleMesh(family={s} size={s} chips={d} cores_per_chip={d} inter_chip_topology=", + .{ + @tagName(self.instance.family), + @tagName(self.instance.size), + self.chips, + self.cores_per_chip, + }, + ); + switch (self.inter_chip_topology) { + .none => try writer.writeAll("none"), + .linear => |size| try writer.print("linear({d})", .{size}), + .ring => |size| try writer.print("ring({d})", .{size}), + .torus2d => |dims| try writer.print("torus2d({d}x{d})", .{ dims.x, dims.y }), + .torus2d_ring => |dims| try writer.print("torus2d_ring({d}x{d}x{d})", .{ dims.x, dims.y, dims.z }), + .point_to_point => |size| try writer.print("point_to_point({d})", .{size}), + } + try writer.writeAll(")"); + } +}; + +pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) !VisibleMesh { + if (nc_ids.len == 0) return error.InvalidDeviceTopology; + + const instance = try neuron.instance(); + const physical_cores_per_chip = try instance.coresPerChip(); + const placements = try allocator.alloc(Placement, nc_ids.len); + errdefer allocator.free(placements); + + for (nc_ids, placements, 0..) |nc_id, *placement, input_index| { + placement.* = .{ + .input_index = input_index, + .nc_id = nc_id, + .chip = chipOf(nc_id, physical_cores_per_chip), + .core = coreOf(nc_id, physical_cores_per_chip), + }; + } + + const Sort = struct { + fn lessThan(_: void, a: Placement, b: Placement) bool { + return if (a.chip == b.chip) a.core < b.core else a.chip < b.chip; + } + }; + std.mem.sort(Placement, placements, {}, Sort.lessThan); + + const first_chip = placements[0].chip; + const last_chip = placements[placements.len - 1].chip; + const selected_chip_count = last_chip - first_chip + 1; + const selected_cores_per_chip = try std.math.divExact(usize, placements.len, selected_chip_count); + const inter_chip_topology = interChipTopology(instance, selected_chip_count); + + var res: VisibleMesh = .{ + .instance = instance, + .chips = selected_chip_count, + .cores_per_chip = selected_cores_per_chip, + .inter_chip_topology = inter_chip_topology, + .placements = placements, + .axes_buffer = undefined, + .axes_len = 0, + }; + + switch (inter_chip_topology) { + .none => {}, + .linear => |size| appendAxis(&res, .chip_x, size, .linear_ring), + .ring => |size| appendAxis(&res, .chip_x, size, .closed_ring), + .torus2d => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + }, + .torus2d_ring => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + appendAxis(&res, .chip_z, dims.z, .closed_ring); + }, + .point_to_point => |size| appendAxis(&res, .chip_x, size, .point_to_point), + } + + // Inter-chip topology only describes NeuronLink fabric between chips. Add a + // local core axis for multi-core chips, and keep single-chip meshes non-empty. + if (res.cores_per_chip > 1 or res.axes_len == 0) { + appendAxis(&res, .core, res.cores_per_chip, .point_to_point); + } + + return res; +} + +pub fn interChipTopology(instance: neuron.Instance, chip_count: usize) InterChipTopology { + if (chip_count == 1) return .none; + + // References for chip-count and fabric-shape inference: + // - Inf1 instance sizes and 1/4/16-chip layouts + // - Inf2 instance sizes and 1/6/12-chip layouts + // - Trn1 / Trn1n 16-chip 2D torus + // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus + // - Trn3 all-to-all / point-to-point scale-up + return switch (instance.family) { + .inf1 => switch (chip_count) { + 4 => .{ .linear = 4 }, + else => .{ .ring = chip_count }, + }, + .inf2, .inf2e => switch (chip_count) { + 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => switch (instance.size) { + .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => .{ .ring = chip_count }, + }, + }, + .trn1, .trn1n => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn3, .trn3pds98 => .{ .point_to_point = chip_count }, + .unknown => .{ .ring = chip_count }, + }; +} + +pub fn chipOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @divFloor(nc_id, physical_cores_per_chip); +} + +pub fn coreOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @mod(nc_id, physical_cores_per_chip); +} + +fn appendAxis(self: *VisibleMesh, fabric: Fabric, size: usize, geometry: Geometry) void { + self.axes_buffer[self.axes_len] = .{ + .fabric = fabric, + .size = size, + .geometry = geometry, + }; + self.axes_len += 1; +} diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..aaa06989 --- /dev/null +++ b/platforms/oneapi/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") + +zig_shared_library( + name = "zmlxoneapi", + main = "zmlxoneapi.zig", + shared_lib_name = "libzmlxoneapi.so.0", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_oneapi//:__subpackages__"], + zigopts = ["-fno-stack-check"], + deps = ["//stdx"], +) + +cc_library( + name = "libzml_oneapi", + hdrs = ["oneapi.h"], + tags = ["manual"], + deps = ["@libzml_oneapi"], +) + +zig_library( + name = "oneapi", + import_name = "platforms/oneapi", + main = "oneapi.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + "@rules_zig//zig/runfiles", + ] + select({ + "//platforms:oneapi.enabled": [ + ":libzml_oneapi", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/oneapi/README.md b/platforms/oneapi/README.md new file mode 100644 index 00000000..d450bba6 --- /dev/null +++ b/platforms/oneapi/README.md @@ -0,0 +1,54 @@ +# EXPERIMENTAL: oneAPI + + +# About + +This directory contains the oneAPI backend for ZML. It is currently in an early stage of development, and only supports a subset of ZML's features. The oneAPI backend is only available on Linux, and requires an Intel GPU. + +The sharding is not yet supported, so only models that fit in the GPU memory can be run. + +# Getting Started + +## Prerequisites + +- An Intel GPU +- bazel see [Getting Started](../README.md#getting-started) for installation instructions. + +### Linux Only + +One GPU is supported on Linux. +ONEAPI_DEVICE_SELECTOR environment variable must be set to select the GPU device. + +For example, to select the first GPU: +```bash +export ONEAPI_DEVICE_SELECTOR=level_zero:0 +``` + +#### Run + + +To run the LLM example with oneAPI and the Llama 3.2 1B model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` + +To run the LLM example with oneAPI and a local model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=path/to/your/model/Llama-3.1-8B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` diff --git a/platforms/oneapi/libzml_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel new file mode 100644 index 00000000..eb08e652 --- /dev/null +++ b/platforms/oneapi/libzml_oneapi.BUILD.bazel @@ -0,0 +1,154 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@zml//bazel:patchelf.bzl", "patchelf") + +patchelf( + name = "libzml_oneapi_so", + src = "lib/libzml_oneapi.so", + add_needed = [ + "libzmlxoneapi.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxoneapi_dlopen", + }, + set_rpath = "$ORIGIN", + soname = "libzml_oneapi.so", +) + +patchelf( + name = "libdnnl_so_3", + src = "@intel-oneapi-dnnl-2026.0//:libdnnl_so_3", + set_rpath = "$ORIGIN", + soname = "libdnnl.so.3", +) + +patchelf( + name = "libtbb_so_12", + src = "@intel-oneapi-tbb-2023.1//:libtbb_so_12", + set_rpath = "$ORIGIN", + soname = "libtbb.so.12", +) + +patchelf( + name = "libsycl_so", + src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:libsycl_so", + set_rpath = "$ORIGIN", +) + +patchelf( + name = "libccl_legacy_so", + src = "@intel-oneapi-ccl-2022.1//:libccl_legacy_so", + set_rpath = "$ORIGIN", +) + +patchelf( + name = "libze_intel_gpu_so", + src = "@libze-intel-gpu1//:libze_intel_gpu_so", + set_rpath = "$ORIGIN", + soname = "libze_intel_gpu.so.1", +) + +patchelf( + name = "libigc_so_2", + src = "@libigc2//:libigc_so_2", + set_rpath = "$ORIGIN", + soname = "libigc.so.2", +) + +patchelf( + name = "libiga64_so_2", + src = "@libigc2//:libiga64_so_2", + set_rpath = "$ORIGIN", + soname = "libiga64.so.2", +) + +patchelf( + name = "libigdfcl_so_2", + src = "@libigdfcl2//:libigdfcl_so_2", + set_rpath = "$ORIGIN", + soname = "libigdfcl.so.2", +) + +ONEAPI_VERSION = "2026.1" + +ONEAPI_CCL_LIB = "ccl/2022.1/lib" + +ONEAPI_MPI_LIB = "mpi/2021.18/lib" + +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" + +ONEAPI_TCM_VERSION = "1.5" + +ONEAPI_UMF_VERSION = "1.1" + +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) + +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) + +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libdnnl_so_3", + ":libiga64_so_2", + ":libigc_so_2", + ":libigdfcl_so_2", + ":libsycl_so", + ":libtbb_so_12", + ":libze_intel_gpu_so", + ":libzml_oneapi_so", + "@intel-oneapi-ccl-2022.1//:ccl_runtime", + "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", + "@intel-oneapi-compiler-shared-runtime-2026.1//:compiler_runtime", + "@intel-oneapi-mkl-core-2026.1//:mkl_core_runtime", + "@intel-oneapi-mkl-sycl-blas-2026.1//:mkl_sycl_blas", + "@intel-oneapi-mkl-sycl-dft-2026.1//:mkl_sycl_dft", + "@intel-oneapi-mkl-sycl-lapack-2026.1//:mkl_sycl_lapack", + "@intel-oneapi-mkl-sycl-rng-2026.1//:mkl_sycl_rng", + "@intel-oneapi-mkl-sycl-sparse-2026.1//:mkl_sycl_sparse", + "@intel-oneapi-mpi-2021.18//:mpi_runtime", + "@intel-oneapi-tcm-1.5//:hwloc", + "@intel-oneapi-umf-1.1//:umf", + "@libigdfcl2//:libopencl_clang", + "@libigdgmm12", + "@libnl-3-200//:libnl_3", + "@libnl-genl-3-200//:libnl_genl_3", + "@libnl-route-3-200//:libnl_route_3", + "@libnuma1", + "@libze1", + "@zlib1g", + "@zml//platforms/oneapi:zmlxoneapi", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + hardlink = "auto", + include_external_repositories = ["**"], + replace_prefixes = { + "libiga64_so_2": "lib", + "libigdfcl_so_2": "lib", + "libigc_so_2": "lib", + "libdnnl_so_3": "lib", + "libzml_oneapi_so": "lib", + "libsycl_so": "lib", + "libtbb_so_12": "lib", + ONEAPI_CCL_LIB: "lib", + ONEAPI_COMPILER_LIB: "lib", + ONEAPI_MPI_LIB: "lib", + ONEAPI_MPI_LIBFABRIC_LIB: "lib", + ONEAPI_MKL_LIB: "lib", + ONEAPI_TCM_LIB: "lib", + ONEAPI_UMF_LIB: "lib", + "lib/x86_64-linux-gnu": "lib", + "libze_intel_gpu_so": "lib", + "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", + }, + root_paths = ["."], +) + +cc_library( + name = "libzml_oneapi", + data = [":sandbox"], + visibility = ["@zml//platforms/oneapi:__subpackages__"], +) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl new file mode 100644 index 00000000..a3c968d3 --- /dev/null +++ b/platforms/oneapi/oneapi.bzl @@ -0,0 +1,325 @@ +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") + +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" + +ONEAPI_VERSION = "2026.1" + +# www.intel.com/content/www/us/en/developer/tools/oneapi/onednn-download.html +ONEAPI_DNNL_VERSION = "2026.0" + +# https://github.com/uxlfoundation/onetbb +ONEAPI_TBB_VERSION = "2023.1" +ONEAPI_TCM_VERSION = "1.5" +ONEAPI_UMF_VERSION = "1.1" + +ONEAPI_CCL_LIB = "ccl/2022.1/lib" +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_DNNL_LIB = "dnnl/{}/lib".format(ONEAPI_DNNL_VERSION) +ONEAPI_MPI_LIB = "mpi/2021.18/lib" +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) +ONEAPI_TBB_LIB = "tbb/{}/lib".format(ONEAPI_TBB_VERSION) +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) + +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) +""" + +_UBUNTU_STRIP_PREFIX = "./usr" +_ONEAPI_STRIP_PREFIX = "./opt/intel/oneapi" + +def _read_packages(mctx, labels): + ret = {} + for label in labels: + data = json.decode(mctx.read(Label(label))) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg + return ret + +_UBUNTU_PACKAGES = { + "libnl-3-200": """ +filegroup( + name = "libnl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-3.so.200"], +)""", + "libnl-genl-3-200": """ +filegroup( + name = "libnl_genl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-genl-3.so.200"], +)""", + "libnl-route-3-200": """ +filegroup( + name = "libnl_route_3", + srcs = ["lib/x86_64-linux-gnu/libnl-route-3.so.200"], +)""", + "libnuma1": """ +filegroup( + name = "libnuma1", + srcs = ["lib/x86_64-linux-gnu/libnuma.so.1"], +)""", + "libigdgmm12": """ +filegroup( + name = "libigdgmm12", + srcs = ["lib/x86_64-linux-gnu/libigdgmm.so.12"], +)""", + "libigc2": """ +filegroup( + name = "libigc_so_2", + srcs = ["lib/x86_64-linux-gnu/libigc.so.2.41.15+0"], +) +filegroup( + name = "libiga64_so_2", + srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.41.15+0"], +) +filegroup( + name = "libigc2", + srcs = [ + ":libiga64_so_2", + ":libigc_so_2", + ], +)""", + "libigdfcl2": """ +filegroup( + name = "libigdfcl_so_2", + srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.41.15+0"], +) +filegroup( + name = "libopencl_clang", + srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.22.1"], +) +filegroup( + name = "libigdfcl2", + srcs = [ + ":libigdfcl_so_2", + ":libopencl_clang", + ], +)""", + "libze-intel-gpu1": """ +filegroup( + name = "libze_intel_gpu_so", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +) +filegroup( + name = "libze_intel_gpu", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +)""", + "libze1": """ +filegroup( + name = "libze1", + srcs = [ + "lib/x86_64-linux-gnu/libze_loader.so.1", + "lib/x86_64-linux-gnu/libze_tracing_layer.so.1", + "lib/x86_64-linux-gnu/libze_validation_layer.so.1", + ], +)""", + "zlib1g": """ +filegroup( + name = "zlib1g", + srcs = ["lib/x86_64-linux-gnu/libz.so.1"], +)""", +} + +_ONEAPI_PACKAGES = { + "intel-oneapi-ccl-2022.1": """ +filegroup( + name = "libccl_legacy_so", + srcs = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], +) +filegroup( + name = "ccl_runtime", + srcs = [ + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0.1", + "{ONEAPI_CCL_LIB}/ccl/kernels/kernels.spv", + "{ONEAPI_CCL_LIB}/libccl.so", + "{ONEAPI_CCL_LIB}/libccl.so.1", + "{ONEAPI_CCL_LIB}/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/libccl.so.2", + "{ONEAPI_CCL_LIB}/libccl.so.2.0", + "{ONEAPI_CCL_LIB}/libccl_legacy_cpu.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0.1", + ], +)""".format(ONEAPI_CCL_LIB = ONEAPI_CCL_LIB), + "intel-oneapi-mpi-2021.18": """ +filegroup( + name = "mpi_runtime", + srcs = [ + "{ONEAPI_MPI_LIB}/libmpi.so", + "{ONEAPI_MPI_LIB}/libmpi.so.12", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpi_abi.so", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4.1", + "{ONEAPI_MPI_LIB}/libmpi_shm_heap_proxy.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort.so", + "{ONEAPI_MPI_LIB}/libmpifort.so.12", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpijava.so", + "{ONEAPI_MPI_LIB}/libmpijava.so.1", + "{ONEAPI_MPI_LIB}/libmpijava.so.1.0", + "{ONEAPI_MPI_LIB}/mpi/libmpi_ze_hooks.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so.1", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libefa-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libmlx-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsm3-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsmx2-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/librxm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libshm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libtcp-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.1-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.12-fi.so", + ], +)""".format( + ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, + ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, + ), + "intel-oneapi-dnnl-2026.0": """ +filegroup( + name = "libdnnl_so_3", + srcs = ["{ONEAPI_DNNL_LIB}/libdnnl.so.3.11"], +)""".format(ONEAPI_DNNL_LIB = ONEAPI_DNNL_LIB), + "intel-oneapi-tbb-2023.1": """ +filegroup( + name = "libtbb_so_12", + srcs = ["{ONEAPI_TBB_LIB}/libtbb.so.12.19"], +)""".format(ONEAPI_TBB_LIB = ONEAPI_TBB_LIB), + "intel-oneapi-tcm-1.5": """ +filegroup( + name = "hwloc", + srcs = ["{ONEAPI_TCM_LIB}/libhwloc.so.15"], +)""".format(ONEAPI_TCM_LIB = ONEAPI_TCM_LIB), + "intel-oneapi-umf-1.1": """ +filegroup( + name = "umf", + srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], +)""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), + "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1": """ +filegroup( + name = "libsycl_so", + srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.9"], +) +filegroup( + name = "sycl_runtime", + srcs = [ + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero_v2.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_opencl.so.0", + "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", + ], +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + "intel-oneapi-compiler-shared-runtime-2026.1": """ +filegroup( + name = "compiler_runtime", + srcs = [ + "{ONEAPI_COMPILER_LIB}/libOpenCL.so", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1.0.0", + "{ONEAPI_COMPILER_LIB}/libimf.so", + "{ONEAPI_COMPILER_LIB}/libintlc.so.5", + "{ONEAPI_COMPILER_LIB}/libirc.so", + "{ONEAPI_COMPILER_LIB}/libirng.so", + "{ONEAPI_COMPILER_LIB}/libsvml.so", + ], +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + "intel-oneapi-mkl-core-2026.1": """ +filegroup( + name = "mkl_core_runtime", + srcs = [ + "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", + "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", + "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", + ], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-blas-2026.1": """ +filegroup( + name = "mkl_sycl_blas", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-dft-2026.1": """ +filegroup( + name = "mkl_sycl_dft", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-lapack-2026.1": """ +filegroup( + name = "mkl_sycl_lapack", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-rng-2026.1": """ +filegroup( + name = "mkl_sycl_rng", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-sparse-2026.1": """ +filegroup( + name = "mkl_sycl_sparse", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_sparse.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), +} + +_ROOT_MODULE_DIRECT_DEPS = ["libzml_oneapi"] + +def _oneapi_impl(mctx): + loaded_packages = _read_packages(mctx, [ + "@zml//platforms/oneapi:packages.lock.json", + ]) + + http_archive( + name = "libzml_oneapi", + build_file = "libzml_oneapi.BUILD.bazel", + sha256 = _PLUGIN_ONEAPI_SHA256, + url = _PLUGIN_ONEAPI_URL, + ) + + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _UBUNTU_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + for pkg_name, build_file_content in _ONEAPI_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _ONEAPI_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = _ROOT_MODULE_DIRECT_DEPS, + root_module_direct_dev_deps = [], + ) + +oneapi_packages = module_extension( + implementation = _oneapi_impl, +) diff --git a/platforms/oneapi/oneapi.h b/platforms/oneapi/oneapi.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/oneapi/oneapi.h @@ -0,0 +1 @@ + #include diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..c5654741 --- /dev/null +++ b/platforms/oneapi/oneapi.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/oneapi"); + +pub fn isEnabled() bool { + return platforms_options.oneapi_enabled; +} + +fn hasOneApiDevice(io: std.Io) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (std.mem.startsWith(u8, entry.name, "renderD")) { + return true; + } + } + + return false; +} + +fn setupOneAPIEnv(driver_path: [:0]const u8) void { + _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); + _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); + _ = c.setenv("FI_PROVIDER", "shm", 1); + // collective-permute through oneccl recv/send often relies on sycl d2d memcpy underneath, + // that doesn't work because of peer residency likely this: + // https://github.com/intel/compute-runtime/issues/953 + _ = c.setenv("XLA_ONECCL_COLLECTIVE_PERMUTE_BYPASS_SYCL_P2P", "1", 1); + _ = c.setenv("ONEAPI_DEVICE_SELECTOR", "level_zero:*", 0); + _ = c.setenv("ZE_ENABLE_ALT_DRIVERS", driver_path, 0); +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + + if (!hasOneApiDevice(io)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_oneapi/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for oneAPI runtime", .{}); + return error.FileNotFound; + }; + + var driver_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const driver_path = try stdx.Io.Dir.path.bufJoinZ(&driver_path_buf, &.{ sandbox_path, "lib", "libze_intel_gpu.so.1" }); + setupOneAPIEnv(driver_path); + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_oneapi.so" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json new file mode 100644 index 00000000..140ce102 --- /dev/null +++ b/platforms/oneapi/packages.lock.json @@ -0,0 +1,269 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-ccl-2022.1_2022.1.2-9_amd64", + "name": "intel-oneapi-ccl-2022.1", + "sha256": "432e62469978f60a9b35b55216cfd495b0adc76c334707d3bbc81f69b7cc2948", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.2-9_amd64.deb" + ], + "version": "2022.1.2-9" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mpi-2021.18_2021.18.1-9_amd64", + "name": "intel-oneapi-mpi-2021.18", + "sha256": "f6f76f39755329bbac890deeb72fb105df064dc5dcb67cde1396e4d6ef0f6706", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.1-9_amd64.deb" + ], + "version": "2021.18.1-9" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.1-325_amd64", + "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1", + "sha256": "df0413d57cff5a9c67bd02c1e958d7e0512f77878bbcf976c9e08af7c0b1069c", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.1-325_amd64.deb" + ], + "version": "2026.1.1-325" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.1-325_amd64", + "name": "intel-oneapi-compiler-shared-runtime-2026.1", + "sha256": "c0c77700243af0413f7fbefd26649943c4959171ead91b6ace36ef0c8dfb2630", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.1-325_amd64.deb" + ], + "version": "2026.1.1-325" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-dnnl-2026.0_2026.0.2-40_amd64", + "name": "intel-oneapi-dnnl-2026.0", + "sha256": "4b0f7e23415a695f204f8080328977225759534158d813c2225403f8851433e8", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-dnnl-2026.0-2026.0.2-40_amd64.deb" + ], + "version": "2026.0.2-40" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-core-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-core-2026.1", + "sha256": "3b230c02fc0ac44f8f71062c20cd883b37c6c1e00940dc0a7f13b02d9c47bd93", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-blas-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-blas-2026.1", + "sha256": "9668d51a7cf68ae4b8944a8cdaa24f6d4ef72604c048f68fd8603b2b792bd783", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-dft-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-dft-2026.1", + "sha256": "31862dd325a31ccacc1e5e9c93140f4c8bd8a0353e0b3658e2b87e329e58d373", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-lapack-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-lapack-2026.1", + "sha256": "f701832f5e49b29731c15a27a094122bebe6a1fbd90ddf14c431b7923b8fd921", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-rng-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-rng-2026.1", + "sha256": "b0748e152f2042dedd1d21c784fb2a8b551c573b7e7ed0361ada2423deb275bf", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-sparse-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-sparse-2026.1", + "sha256": "6a499dc1c5f049ad522a5288151ff0c93f535b8e8ab34edcd773785338e07d45", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tbb-2023.1_2023.1.0-151_amd64", + "name": "intel-oneapi-tbb-2023.1", + "sha256": "f6e4ae469436a8c5ee2250203268344164c9ff455a2207219c75c463aab99a8b", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tbb-2023.1-2023.1.0-151_amd64.deb" + ], + "version": "2023.1.0-151" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tcm-1.5_1.5.0-489_amd64", + "name": "intel-oneapi-tcm-1.5", + "sha256": "7122ed7748d54342b24f0a2838a64f24b54e430ac4d06b2961a8eefee4ffb089", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tcm-1.5-1.5.0-489_amd64.deb" + ], + "version": "1.5.0-489" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-umf-1.1_1.1.0-340_amd64", + "name": "intel-oneapi-umf-1.1", + "sha256": "8705a997042c76f67228b4135184754e1cd3f8f68b6629bcb261c1d5b958ce78", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-umf-1.1-1.1.0-340_amd64.deb" + ], + "version": "1.1.0-340" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-3-200", + "sha256": "15b3f61db753dc3db6f62f4259f8b822299343e500886776486612a13a2ed247", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-genl-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-genl-3-200", + "sha256": "9ca0cb63f295d00e6346b9924278c07cfc61311cbd288b805b06e25aa08c9fd8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-genl-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-route-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-route-3-200", + "sha256": "c9dbd29cb7fdcf1a0587726cbc5432a7d3e81242108dfcfa2dd6716c44c74d4d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-route-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnuma1_2.0.18-1ubuntu0.24.04.1_amd64", + "name": "libnuma1", + "sha256": "f333b8edf6f0b705c19f2a67008194df66f7aab0fa2350dc98510569a033ff29", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/n/numactl/libnuma1_2.0.18-1ubuntu0.24.04.1_amd64.deb" + ], + "version": "2.0.18-1ubuntu0.24.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdgmm12_22.10.1-1_24.04_ppa1_amd64", + "name": "libigdgmm12", + "sha256": "2c68fcb150cba1b679063266cdf0054f26f18af90de71d6d3b614eaf95c98f80", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.1-1~24.04~ppa1_amd64.deb" + ], + "version": "22.10.1-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigc2_2.41.15-1_24.04_amd64", + "name": "libigc2", + "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + ], + "version": "2.41.15-1~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdfcl2_2.41.15-1_24.04_amd64", + "name": "libigdfcl2", + "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + ], + "version": "2.41.15-1~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze-intel-gpu1_26.31.39395.14-1_24.04_ppa1_amd64", + "name": "libze-intel-gpu1", + "sha256": "39f10740e40cf0de1015997d0de094a3c3dc6371c3b9ea1dcd15771d64559c14", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.31.39395.14-1~24.04~ppa1_amd64.deb" + ], + "version": "26.31.39395.14-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze1_1.32.0-1_24.04_ppa1_amd64", + "name": "libze1", + "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + ], + "version": "1.32.0-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g", + "sha256": "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" + ], + "version": "1:1.3.dfsg-3.1ubuntu2.1" + } + ], + "version": 1 +} diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml new file mode 100644 index 00000000..5a829ac5 --- /dev/null +++ b/platforms/oneapi/packages.yaml @@ -0,0 +1,53 @@ +# +# bazel run @apt_oneapi//:lock +# +version: 1 + +sources: + - channel: all main + url: https://apt.repos.intel.com/oneapi + - channel: noble main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-security main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-updates main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble main + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z + + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. + - channel: noble main + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z + +archs: + - "amd64" + +packages: + # Do not use "intel-oneapi-toolkit" package + # It will install an opaque deb file that will at preinst download the sdk toolkit + - "intel-oneapi-ccl-2022.1" + - "intel-oneapi-mpi-2021.18" + - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1" + - "intel-oneapi-compiler-shared-runtime-2026.1" + - "intel-oneapi-dnnl-2026.0" + - "intel-oneapi-mkl-core-2026.1" + - "intel-oneapi-mkl-sycl-blas-2026.1" + - "intel-oneapi-mkl-sycl-dft-2026.1" + - "intel-oneapi-mkl-sycl-lapack-2026.1" + - "intel-oneapi-mkl-sycl-rng-2026.1" + - "intel-oneapi-mkl-sycl-sparse-2026.1" + - "intel-oneapi-tbb-2023.1" + - "intel-oneapi-tcm-1.5" + - "intel-oneapi-umf-1.1" + - "libnl-3-200" + - "libnl-genl-3-200" + - "libnl-route-3-200" + - "libnuma1" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" + - "libigc2 (= 2.41.15-1~24.04)" + - "libigdfcl2 (= 2.41.15-1~24.04)" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" + - "libze1" + - "zlib1g" diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig new file mode 100644 index 00000000..6be2505a --- /dev/null +++ b/platforms/oneapi/zmlxoneapi.zig @@ -0,0 +1,31 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libOpenCL.so", "libOpenCL.so.1" }, + .{ "libze_loader.so", "libze_loader.so.1" }, + .{ "libumf.so", "libumf.so.1" }, + .{ "libur_loader.so", "libur_loader.so.0" }, + .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, + .{ "libur_adapter_level_zero_v2.so", "libur_adapter_level_zero_v2.so.0" }, + .{ "libur_adapter_opencl.so", "libur_adapter_opencl.so.0" }, + .{ "libsycl.so", "libsycl.so.9" }, + .{ "libmkl_core.so", "libmkl_core.so.3" }, + .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.3" }, + .{ "libmkl_sequential.so", "libmkl_sequential.so.3" }, + .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.6" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} diff --git a/platforms/packages.bzl b/platforms/packages.bzl index 633f98b1..6d3117cd 100644 --- a/platforms/packages.bzl +++ b/platforms/packages.bzl @@ -45,10 +45,8 @@ def _read(mctx, labels): ret = {} for label in labels: data = json.decode(mctx.read(Label(label))) - ret.update({ - pkg["name"]: pkg - for pkg in data["packages"] - }) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg return ret packages = struct( diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 4033415c..789c9a10 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -2,35 +2,49 @@ const std = @import("std"); const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); +const furiosa = @import("platforms/furiosa"); +const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); +const oneapi = @import("platforms/oneapi"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const rocm = @import("platforms/rocm"); const tpu = @import("platforms/tpu"); +const platforms = @This(); + pub const Platform = enum { cpu, cuda, rocm, tpu, neuron, + oneapi, + metal, + furiosa, + + pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { + inline for (@typeInfo(Platform).@"enum".field_names) |field| { + const target: Platform = @field(Platform, field); + if (comptime (target != .cpu and target.isEnabled())) return target; + } + + if (!comptime Platform.cpu.isEnabled()) @compileError("No platform was enabled, use --@zml//platforms:cuda=true to eg enable Cuda"); + return .cpu; + } + + pub fn isEnabled(target: Platform) bool { + return switch (target) { + inline else => |tag| @field(platforms_options, @tagName(tag) ++ "_enabled"), + }; + } + + pub fn load(target: Platform, allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + return switch (target) { + inline else => |tag| @field(platforms, @tagName(tag)).load(allocator, io), + }; + } }; -pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { - return switch (tag) { - .cpu => try cpu.load(allocator, io), - .cuda => try cuda.load(allocator, io), - .rocm => try rocm.load(allocator, io), - .tpu => try tpu.load(allocator, io), - .neuron => try neuron.load(allocator, io), - }; -} - -pub fn isEnabled(tag: Platform) bool { - return switch (tag) { - .cpu => cpu.isEnabled(), - .cuda => cuda.isEnabled(), - .rocm => rocm.isEnabled(), - .tpu => tpu.isEnabled(), - .neuron => neuron.isEnabled(), - }; -} +pub const load = Platform.load; +pub const isEnabled = Platform.isEnabled; diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 76ee4e9f..91651372 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -1,47 +1,49 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") +exports_files( + ["merge_amdgpu_ids.zig"], + visibility = ["@libzml_rocm//:__subpackages__"], +) + zig_shared_library( name = "zmlxrocm", main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", - visibility = ["@libpjrt_rocm//:__subpackages__"], + target_compatible_with = ["@platforms//os:linux"], + visibility = [ + "//bin/zml-smi:__pkg__", + "@libzml_rocm//:__subpackages__", + ], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], deps = ["//stdx"], ) -alias( - name = "hipblaslt", - actual = "@libpjrt_rocm//:hipblaslt", -) - cc_library( - name = "empty", - defines = ["ZML_RUNTIME_ROCM_DISABLED"], -) - -cc_library( - name = "libpjrt_rocm", - defines = ["ZML_RUNTIME_ROCM"], - deps = ["@libpjrt_rocm"], + name = "libzml_rocm", + hdrs = ["rocm.h"], + tags = ["manual"], + deps = ["@libzml_rocm"], ) zig_library( name = "rocm", import_name = "platforms/rocm", main = "rocm.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", + "//platforms:options", ] + select({ "//platforms:rocm.enabled": [ - ":libpjrt_rocm", + ":libzml_rocm", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel deleted file mode 100644 index 4aaabacf..00000000 --- a/platforms/rocm/libpjrt_rocm.BUILD.bazel +++ /dev/null @@ -1,98 +0,0 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@aspect_bazel_lib//lib:tar.bzl", "tar") -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") -load("@zml//bazel:patchelf.bzl", "patchelf") - -bool_flag( - name = "hipblaslt", - build_setting_default = False, - visibility = ["//visibility:public"], -) - -config_setting( - name = "_hipblaslt", - flag_values = {":hipblaslt": "True"}, -) - -patchelf( - name = "libpjrt_rocm_so", - src = "libpjrt_rocm.so", - add_needed = [ - "libzmlxrocm.so.0", - ], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - set_rpath = "$ORIGIN", -) - -copy_to_directory( - name = "sandbox", - srcs = [ - ":libpjrt_rocm_so", - "@comgr//:amd_comgr", - "@hip-runtime-amd//:amdhip", - "@hip-runtime-amd//:hiprtc", - "@hipblaslt//:hipblaslt", - "@hipblaslt//:rocroller", - "@hipfft", - "@hipsolver", - "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", - "@hsa-rocr//:hsa-runtime", - "@libdrm-amdgpu-amdgpu1", - "@libdrm-amdgpu-common//:amdgpu_ids", - "@libdrm2-amdgpu", - "@libelf1", - "@libnuma1", - "@libtinfo6", - "@libzstd1", - "@miopen-hip//:MIOpen", - "@rccl", - "@rocblas//:rocblas", - "@rocblas//:runfiles", - "@rocfft", - "@rocm-core", - "@rocm-device-libs//:runfiles", - "@rocm-smi-lib//:rocm_smi", - "@rocprofiler-register", - "@rocprofiler-sdk//:rocprofiler-sdk", - "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", - "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", - "@rocsolver", - "@roctracer", - "@roctracer//:roctx", - "@zlib1g", - "@zml//platforms/rocm:zmlxrocm", - ] + select({ - ":_hipblaslt": ["@hipblaslt//:runfiles"], - "//conditions:default": [], - }), - replace_prefixes = { - "amdhip": "lib", - "hipblaslt": "lib", - "lib/x86_64-linux-gnu": "lib", - "libdrm-amdgpu-amdgpu1": "lib", - "libelf1": "lib", - "libpjrt_rocm_so": "lib", - "opt/amdgpu/lib/x86_64-linux-gnu": "lib", - "opt/amdgpu/share": "share", - "rocblas": "lib", - "rocroller": "lib", - "platforms/rocm": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - }, - add_directory_to_runfiles = True, - include_external_repositories = ["**"], -) - -tar( - name = "amdgpu_ids_layer", - srcs = ["@libdrm-amdgpu-common//:amdgpu_ids"], - visibility = ["//visibility:public"], -) - -cc_library( - name = "libpjrt_rocm", - data = [":sandbox"], - visibility = ["@zml//platforms/rocm:__subpackages__"], -) diff --git a/platforms/rocm/libzml_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel new file mode 100644 index 00000000..7996b001 --- /dev/null +++ b/platforms/rocm/libzml_rocm.BUILD.bazel @@ -0,0 +1,122 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@zml//bazel:patchelf.bzl", "patchelf") + +patchelf( + name = "libzml_rocm_so", + src = "lib/libzml_rocm.so", + add_needed = [ + "libzmlxrocm.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", +) + +zig_binary( + name = "merge_amdgpu_ids", + main = "@zml//platforms/rocm:merge_amdgpu_ids.zig", +) + +run_binary( + name = "amdgpu_ids", + srcs = [ + "@libdrm-common//:amdgpu_ids", + "@libdrm_mesa_amdgpu_ids//file:amdgpu.ids", + ], + outs = ["amdgpu.ids"], + args = [ + "$(location @libdrm-common//:amdgpu_ids)", + "$(location @libdrm_mesa_amdgpu_ids//file:amdgpu.ids)", + "$(location amdgpu.ids)", + ], + tool = ":merge_amdgpu_ids", + visibility = ["@zml//bin/zml-smi:__pkg__"], +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":amdgpu_ids", + ":libzml_rocm_so", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-base//:rocm-core", + "@amdrocm-base//:rocprofiler-register", + "@amdrocm-blas//:hipblaslt", + "@amdrocm-blas//:hipblaslt_runfiles", + "@amdrocm-blas//:rocblas", + "@amdrocm-blas//:rocblas_runfiles", + "@amdrocm-dnn//:MIOpen", + "@amdrocm-fft//:hipfft", + "@amdrocm-fft//:rocfft", + "@amdrocm-llvm//:llvm_libs", + "@amdrocm-llvm//:rocm_device_libs_runfiles", + "@amdrocm-math-common//:host_math_libs", + "@amdrocm-profiler-base//:hsa-amd-aqlprofile", + "@amdrocm-profiler-base//:rocprofiler-sdk", + "@amdrocm-profiler-base//:rocprofv3_tool", + "@amdrocm-profiler-base//:roctracer", + "@amdrocm-profiler-base//:roctx", + "@amdrocm-rccl//:rccl", + "@amdrocm-runtime//:amd_comgr", + "@amdrocm-runtime//:amdhip", + "@amdrocm-runtime//:hiprtc", + "@amdrocm-runtime//:hsa-runtime", + "@amdrocm-solver//:hipsolver", + "@amdrocm-solver//:rocsolver", + "@amdrocm-sparse//:rocsparse", + "@amdrocm-sysdeps//:system_libs", + "@libatomic1", + "@llvm-libunwind1//:libunwind", + "@zml//platforms/rocm:zmlxrocm", + ], + add_directory_to_runfiles = True, + include_external_repositories = ["**"], + replace_prefixes = { + "MIOpen": "lib", + "amdhip_so": "lib", + "amdhip": "lib", + "amd_comgr": "lib", + "hipdnn_backend": "lib", + "hipblas": "lib", + "hiprtc_so": "lib", + "hipfft": "lib", + "hiprtc": "lib", + "hipsolver": "lib", + "hipsparse": "lib", + "hipsparselt": "lib", + "hsa-amd-aqlprofile": "lib", + "hsa-runtime": "lib", + "hipblaslt": "lib", + "lib/host-math/lib": "lib", + "lib/llvm/lib": "lib", + "lib/llvm/amdgcn": "amdgcn", + "libzml_rocm_so": "lib", + "amdgpu.ids": "share/libdrm/amdgpu.ids", + "rccl": "lib", + "rocblas": "lib", + "rocfft": "lib", + "rocm-core": "lib", + "rocprofiler-register": "lib", + "rocprofiler-sdk_so": "lib", + "rocprofiler-sdk": "lib", + "rocm_smi": "lib", + "rocsolver": "lib", + "rocroller": "lib", + "rocsparse": "lib", + "roctracer": "lib", + "roctx": "lib", + "platforms/rocm": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, + visibility = ["//visibility:public"], +) + +cc_library( + name = "libzml_rocm", + data = [":sandbox"], + visibility = ["@zml//platforms/rocm:__subpackages__"], +) diff --git a/platforms/rocm/merge_amdgpu_ids.zig b/platforms/rocm/merge_amdgpu_ids.zig new file mode 100644 index 00000000..b939df82 --- /dev/null +++ b/platforms/rocm/merge_amdgpu_ids.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const arena = init.arena.allocator(); + + const args = try init.minimal.args.toSlice(arena); + + if (args.len != 4) { + try std.Io.File.stderr().writeStreamingAll(io, "Usage: merge_amdgpu_ids \n"); + std.process.exit(1); + } + + var entries: std.StringHashMapUnmanaged([]const u8) = .empty; + defer entries.deinit(arena); + + const cwd = std.Io.Dir.cwd(); + + try parseFile(io, cwd, arena, args[2], &entries); + try parseFile(io, cwd, arena, args[1], &entries); + + const out = try cwd.createFile(io, args[3], .{}); + defer out.close(io); + + var buf: [4096]u8 = undefined; + var writer = out.writer(io, &buf); + + try writer.interface.print("# List of AMDGPU IDs\n" ++ + "#\n" ++ + "# Syntax:\n" ++ + "# device_id,\trevision_id,\tproduct_name <-- single tab after comma\n" ++ + "\n" ++ + "1.0.0\n", .{}); + + var it = entries.valueIterator(); + while (it.next()) |v| { + try writer.interface.print("{s}\n", .{v.*}); + } + + try writer.flush(); +} + +fn parseFile( + io: std.Io, + cwd: std.Io.Dir, + arena: std.mem.Allocator, + path: []const u8, + entries: *std.StringHashMapUnmanaged([]const u8), +) !void { + const file = try cwd.openFile(io, path, .{ .mode = .read_only }); + defer file.close(io); + + var buf: [4096]u8 = undefined; + var reader = file.reader(io, &buf); + + while (true) { + const raw = try reader.interface.takeDelimiter('\n') orelse break; + + const line = std.mem.trimEnd(u8, raw, " \t\r"); + if (line.len == 0 or line[0] == '#') continue; + + var iter = std.mem.splitSequence(u8, line, ","); + + const device = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const revision = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const product = std.mem.trim(u8, iter.rest(), " \t"); + + const key = try std.fmt.allocPrint(arena, "{s},{s}", .{ device, revision }); + const value = try std.fmt.allocPrint(arena, "{s},\t{s},\t{s}", .{ device, revision, product }); + + try entries.put(arena, key, value); + } +} diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json old mode 100755 new mode 100644 index 67139f24..2b2a4836 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -3,49 +3,6 @@ { "arch": "amd64", "dependencies": [ - { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" - } - ], - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-common", - "sha256": "baa11bc7e3f8e6931a69dea7f1485c2f982da184811ac7545342953ffb64abc4", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_all.deb" - ], - "version": "1.0.0.70200-2278374.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "sha256": "916fdee8e388c36a67abda89e0868e5789f4b77a3c351f945db4218fbcbe226c", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70200-2278374.22.04_all.deb" - ], - "version": "1:7.2.70200-2278374.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libc6_2.35-0ubuntu3.10_amd64", "name": "libc6", @@ -56,508 +13,24 @@ "name": "libcrypt1", "version": "1:4.4.27-1" }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70200-2278374.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70200-2278374.22.04" - } - ], - "key": "hsa-rocr_1.18.0.70200-43_22.04_amd64", - "name": "hsa-rocr", - "sha256": "c2ee2205b61a0c6b00478e70fa0cde4441fc9658cddcedb8bff07ab07569719c", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70200-43~22.04_amd64.deb" - ], - "version": "1.18.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", "name": "libgcc-s1", "version": "12.3.0-1ubuntu1~22.04" }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, { "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", "name": "gcc-12-base", "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" } ], - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "sha256": "144d516c55f368e82c231df469c1d3097ee4f44302ef3b71fe3eeded4199f5d3", + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260613092327-p-e80beda6e255-1_exp1_20260613092437.81_amd64", + "name": "llvm-libunwind1", + "sha256": "0da2b5596faee4e53456b7bdf51458e7bd5ec06852af93f2538e74c122032df2", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70200-43~22.04_amd64.deb" + "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81_amd64.deb" ], - "version": "0.6.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" + "version": "1:22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81" }, { "arch": "amd64", @@ -581,6 +54,17 @@ ], "version": "1:4.4.27-1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, { "arch": "amd64", "dependencies": [], @@ -594,14 +78,294 @@ }, { "arch": "amd64", - "dependencies": [], - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "sha256": "9e8c61b3b260547f88b795f521d82249dfae50f9f82522c6ba9a4fad8fe10f0a", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-core/rocm-core_7.2.0.70200-43~22.04_amd64.deb" + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } ], - "version": "7.2.0.70200-43~22.04" + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "sha256": "35a306712d8b15b30c42ecd73ec087813eb01c0b3125dc8f7ca2b5134e133522", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-common_2.4.113-2~ubuntu0.22.04.1_all.deb" + ], + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", + "name": "amdrocm-amdsmi10.1", + "sha256": "994e715d38a16d9c408eebd508c66123ff2f64cf4d4f4ca15780c4500e85103a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-amdsmi10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "sha256": "a2493d98144a586df49489c4b4b7a7449c9eb123597515929699b1b5fccfd916", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sysdeps10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", @@ -944,6 +708,17 @@ ], "version": "2.5.1-2build2" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, { "arch": "amd64", "dependencies": [], @@ -1032,57 +807,23 @@ ], "version": "3.10.6-1~22.04.1" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" - ], - "version": "0.186-1ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" - ], - "version": "2.0.14-3ubuntu2" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "sha256": "460f6dd499dbff0d4bc368d40b0c3f943dbf981df77f362ac05cce7befcf7779", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70200-2278374.22.04_amd64.deb" - ], - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm2-amdgpu", - "sha256": "a5a620541bcd852d6feb8299dda804c41cb979a3870cf82a783112b58ad9d7e4", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70200-2278374.22.04_amd64.deb" - ], - "version": "1:2.4.125.70200-2278374.22.04" - }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1105,256 +846,18 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "hsa-amd-aqlprofile_1.0.0.70200-43_22.04_amd64", - "name": "hsa-amd-aqlprofile", - "sha256": "c4eabbbac5adb81d30442fe4172660591b4b11f70596035c0cd6b5dc80bd27be", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70200-43~22.04_amd64.deb" - ], - "version": "1.0.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "sha256": "a62cc2bbf9cfc46b263b17ff90bea66ae01267c03014b55a9764eab6895db4a7", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64.deb" - ], - "version": "25.30.13-2281980.22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", "version": "12.3.0-1ubuntu1~22.04" }, { @@ -1363,219 +866,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -1592,6 +920,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -1613,51 +1046,561 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" } ], - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "sha256": "8f58b1958811430ba19426489608b3cbf685e5a78c1d6d9414c7f23cc8311a13", + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "sha256": "a3d48032edee53c51016b28884a1fe81e2d2f84e087c2546dd20753402ad53c9", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/c/comgr/comgr_3.0.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-base10.1_10.1.0-3_amd64.deb" ], - "version": "3.0.0.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "sha256": "2cf4fa3b82f9b37be9fc18ca6886e1791585c1b8041758ccba9e3c5dadb7c6b1", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime10.1_10.1.0-3_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "sha256": "6a4727102d830f6c97b911351192b15623189278afad9b4558514a6fe51ca58a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "sha256": "06faa64479f1987e4f870ff9c1ccfdf3c0c441bd3886136d6e24c11316c9f3d6", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libgcc-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "11.4.0-1ubuntu1~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "sha256": "6bf564c2bb191aefcf96758502a7ac24b2af4d644c87b3678e4b1107c5494b13", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libquadmath0_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "6.3-2ubuntu0.1" + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "sha256": "becb1726d3b8a45cfd336e502fc29d7c43e024689857c670ecf75a5362732c63", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libubsan1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "sha256": "7b14e81317fce5e62d6cc2454d975d3ec4eac90710a1a536d63c9ee84a50b6f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libtsan0_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "sha256": "5448bba72109dbf2df6ee30463b5f06c074138006cd1d5730fb5189b93e6d563", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/gcc-11-base_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "sha256": "606e212c3cbc8fb622313fdb32f78a024b4b5b9fe54b1cc5a8f731297023b899", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/liblsan0_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "sha256": "0580e3209722dd0e6d9e57c4149a0f8f6073f975cf8b5ede26ac54289c2fc99a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libasan6_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "sha256": "ac3c96ed50a42b4f57c410576bdf816710684e3e32e34f704a7295014565b724", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libitm1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "sha256": "072a7ee2dc00bc92a816e11ae62ad5a497c86fc1f0473c0a095f29e13003da72", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgomp1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "sha256": "5ab9c245d4c65a3986d567c54e2503251c92bd369c9f42e48a2c4b1f1835dcf5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libstdc++-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" }, { "arch": "amd64", @@ -1736,29 +1679,43 @@ ], "version": "2.35-0ubuntu3.10" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" - ], - "version": "6.3-2ubuntu0.1" - }, { "arch": "amd64", "dependencies": [ { - "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", - "name": "rocprofiler-sdk-rocpd", - "version": "1.1.0-43~22.04" + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1781,159 +1738,19 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -1941,368 +1758,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", - "name": "rocprofiler-sdk-roctx", - "version": "1.1.0-43~22.04" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - } - ], - "key": "rocprofiler-sdk_1.1.0-43_22.04_amd64", - "name": "rocprofiler-sdk", - "sha256": "adaebb45a2b713fb7677cca10d50afbce33b025f6bbf5a84cf37707fc413127d", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-43~22.04_amd64.deb" - ], - "version": "1.1.0-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", - "name": "rocprofiler-sdk-rocpd", - "sha256": "2dbfad4c5c1e55bb65771bb3aa34460c52e6c7952767f26756d8aa7d98c47414", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-43~22.04_amd64.deb" - ], - "version": "1.1.0-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", - "name": "rocprofiler-sdk-roctx", - "sha256": "4c74438154cd91d81768c5ecbbed24859f787db638dcacce6b70445abf8577c7", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-43~22.04_amd64.deb" - ], - "version": "1.1.0-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocrand_4.2.0.70200-43_22.04_amd64", - "name": "rocrand", - "version": "4.2.0.70200-43~22.04" - }, - { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" - }, - { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" - }, - { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70200-43~22.04" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -2319,6 +1812,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -2340,77 +1938,540 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" }, { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" }, { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" }, { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" } ], - "key": "miopen-hip_3.5.1.70200-43_22.04_amd64", - "name": "miopen-hip", - "sha256": "c9b6ea5b4f07aa6ee21f95f69ac3dd88ef87449e52c41d751b01b071ab121656", + "key": "amdrocm-blas10.1_10.1.0-3_amd64", + "name": "amdrocm-blas10.1", + "sha256": "2a8a63f7c3e5b5e11974433c9cb6abdd297c9939c07cd061f83e0a8d3867e785", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1_10.1.0-3_amd64.deb" ], - "version": "3.5.1.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocrand_4.2.0.70200-43_22.04_amd64", - "name": "rocrand", - "sha256": "2b22a1cc2861eff622c6756b7cba160fb0b9211cbd9452b89b7d3ab3ce19e6c8", + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "sha256": "ba944cc7b634d098344315357b126a6ab7fff39ab34970f798d20a3e8b1177d2", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocrand/rocrand_4.2.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "4.2.0.70200-43~22.04" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "sha256": "089bf10ade33b38698d8eb02af7cd8ec9768ad1946afd2e76b18175d7dbbaa26", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "sha256": "b0714db7f27322097d7d1940e9c95a431d9ec11ff7ea93081ca549f8b72a379c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "sha256": "44faf2440577e65c51b9b6474c1b70a96503a44c7939e6b8ec75512ab928f08e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -2433,159 +2494,29 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", @@ -2593,558 +2524,44 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" - } - ], - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "sha256": "a7baff52908f4a25b134d7e2613c1eb2b51b369509465d2a502b457f3ce3c1b2", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt/hipblaslt_1.2.1.70200-43~22.04_amd64.deb" - ], - "version": "1.2.1.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - } - ], - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "sha256": "4b40ae9c1981979f3fb69e9713c5b9fc1a17f868a83732f38f8fb5e53be3a612", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/roctracer/roctracer_4.1.70200.70200-43~22.04_amd64.deb" - ], - "version": "4.1.70200.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -3161,6 +2578,111 @@ "name": "libtirpc-dev", "version": "1.3.2-2ubuntu0.1" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -3182,111 +2704,9 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" - }, - { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" - } - ], - "key": "rocblas_5.2.0.70200-43_22.04_amd64", - "name": "rocblas", - "sha256": "0581ada80350b0d959e1863af4ff4be3ef31fe9838e941d09b8f0693dda7d23d", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocblas/rocblas_5.2.0.70200-43~22.04_amd64.deb" - ], - "version": "5.2.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -3308,106 +2728,6 @@ "name": "libuuid1", "version": "2.37.2-4ubuntu3.4" }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -3428,11 +2748,6 @@ "name": "readline-common", "version": "8.1.2-1" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -3482,557 +2797,873 @@ "key": "python3-minimal_3.10.6-1_22.04.1_amd64", "name": "python3-minimal", "version": "3.10.6-1~22.04.1" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" } ], - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "sha256": "97f477244025fb3bd8762d8d730d18545cafe080cb938f3f6520aeaff6e05c42", + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "sha256": "5f027faf16f293f66fe42cd794ec6b984377183417f49c49c8c2daf3a5675d23", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.26015.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-profiler-base10.1_10.1.0-3_amd64.deb" ], - "version": "7.2.26015.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "sha256": "6b8bcd3f5d0589a3b6447353c140b4ef676c0fbe319050a0f584fde4aa83f16f", + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "sha256": "1ec2e8677941eadcb4b87f48c5cfae18511b1fc2ac08794d61740843eabba022", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocminfo/rocminfo_1.0.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver-host10.1_10.1.0-3_amd64.deb" ], - "version": "1.0.0.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "sha256": "1e4b3e1fc77bc8431e3f5bf81b9193376c9025dd22b6c8a28f87d448cd3d7eaf", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-math-common10.1_10.1.0-3_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "sha256": "9c35dc27bc27b797f80d42d01cf3d8e042c5f6cc1db66f1a54180d5f21ccf3a3", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-host10.1_10.1.0-3_amd64.deb" ], - "version": "1:3.7.0-6" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "sha256": "f988f42ba63f83ca7e65086a5439ac693a0c76c5a963e871b1d8440c7e892c34", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "0.0~2022.01.22-1ubuntu0.1" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "sha256": "c7964290972e15f3db7d73a91c460332ecbb19fa1c187cbdf5836c73804c2748", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "249.11-0ubuntu3.16" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "sha256": "0dadc1402ca074a627918c52680d77dfe99b22abdad643ad36fdfb964ef3fa16", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx942_10.1.0-3_amd64.deb" ], - "version": "29-1ubuntu1" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "sha256": "a0799f0aad9e9378f245288ce5ae653ff1b4f716962fe0f8d6eb160820fb9870", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "sha256": "f329191044bc2aee16a5eb33ab989f393dfbef42c4705542e4d5a0112e406b8d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "sha256": "200cae46a18b621385e50b191f9acbacceea04b8c7757148442661c65ea58c40", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "sha256": "da0bf3ce9c571e9c2a60cc1964e287ff0c2496ebb156003962212650c8a3a42c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "sha256": "0c7e23e418f4892d6d341d9c89f774477d52e105595204801272f2834be1f40b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "sha256": "54d6ddf65e98722e9e5c0b39909d7ccd689ec273c6d7020f6095ba7274404152", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "sha256": "44db7d9e96ecddd26bd8b069a0a5a89610276c8db859ca7cad9f619f0bf0dd60", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "sha256": "7e76b6e1b314d87b55c268f5b6f0b9e300f571eb34de551524bc6302b9ab276b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "sha256": "d555ef3296a63874af1fa99fc7f8f48e57d932abce3c57ec16b2d4cb24ee027e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "sha256": "1906724e7d338bcbf8013e6f02a96618120d39d5d7c8f3b5a40da185be236bc2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "sha256": "43214b4df226aac08647646f71c13dec6cd3be561f9278f0a9fa36b3a5827337", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "sha256": "4612811ba76d77f1faa19b1f4da9db829a473253058aa2b170a2ee555de26a15", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "sha256": "b4df7e69d7c5c0e6e0ebe262411574153cca723c1ef4e9d445dbb539e1724a16", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "sha256": "757a0c725ebcf7701ced94331d1cb4d615aef9e34849c96a84c3aa9dc0e4a754", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "sha256": "740dd6b691abe24651412c7d5eac34f10a3878eec939759b5dc39f720d530084", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "sha256": "dfaec5607aad9e14282c27c1d1f7440cedbde1d147d2e42c11aada69cfb5c30d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "sha256": "b9cfe52bea768867d858a4b0db6de109b95455c1470528cfb51ba5ef4a62f58d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "sha256": "fac6655f10a9e857c844e0f0c9941213ad543c375335272e97ca9a05c3adf64a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "sha256": "3cb059af8f1ce850cf6fabe3182a1904ada1f2460f638476ad2fd57c43423988", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "sha256": "93cccffec257c00cbc45e0970f980fcefe06b2a55893dda128bd97b1b6be29cb", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "sha256": "2b348c1379a9ffbfafea1d606d47574d6b009fb7e729eeaa586a4bcac19830b5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "sha256": "9de48ee15f185b471fc6b0e836abea882153e7d3cced6e3157a5f091f21ea4cd", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "sha256": "d22416ee83fc82dddf4d094eebe28a1a699b09153ed4d996b7763d8686365ab4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "sha256": "8ab4582f863ceeb2bd0f9bc607786f57d03d34515bed3fb38e4bcaa89b05e8f2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "sha256": "7f2fa46ad609d5cf9b8ef181cf0f76e288880d8916530e33b74df33efe508d37", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "sha256": "ffd9c28d8313d64a83401614d6164e4308ade4f7eca52209206c7974f605e541", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "sha256": "81626b7367523f6bb0ba918aa284c9ddcbe59093fad6110462d76cf51dfe06a5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "sha256": "2bda07dcfbf333c35dd9b063e8eb4436bb653afea0aa0ef23fac58ebc54ae021", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "sha256": "b20f76bd6424de48b4a3d7881bc7ee4dc7c0e060970b237a8db53297b46c2acc", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "sha256": "1089660ba5ab633ac0c6f7acc0da10e39926713e71c581f3e27bb7add04cfa59", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "sha256": "626a668f339768a2842a9679a417009ad5c885e9a727e0e34ff7386ab9262f71", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "sha256": "33d78823a628a6b0bf77f4a36cf977c1d1c816f1aa7e1b55bd2af86370bd3f6c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "sha256": "fcabe97aadc69b9a6f4d006191d0e4fb77df22b734b688763b2c437071c9e11a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "sha256": "736385257ca8d3680f49500667fd7bc3fe483e249e80b550da6386e17ac92cb8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "sha256": "49610fda5782cbf81a0ade0ecea1e4cda4a23ea5f619bbeb19954fd3159bc511", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "sha256": "28470e954ec30f8a6cbfff23b85fcdbc0c486b839f904d61a2c922e3157349d0", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "sha256": "7998acee523f9ac9e2ce6c36514c8343fb9a651629275f0354404e47e080f076", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "sha256": "71d7fdbe3619d77d01db2f7895f73330365ca3e1fb792ccc80f8accfb067db7b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "sha256": "15f0a8421d49acdc19365063b7ce4d3b85102bbfbb517e83bc888e4107061694", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "sha256": "dcf5d14b3781669526b279a40866068d60e4d119aa5b628f38fd69e065fba450", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "sha256": "7e65239be07b34e784e22f1fd98aa1b02a89a713aad05c3640ee88b08cf38a7d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "sha256": "028a6598a0f90384b08bdfb0ee42ca2031206e8bf5b65730eeef2a77c201cf87", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "sha256": "5e0397e891522100a5432d433039e8877479b5111f7226401ed185700a56d2ef", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "sha256": "2763f459121dcca48edc4997dc861763cb3c237d102a0cf771209f5d4e0865db", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "sha256": "2aed6082214e2ef53cc472d169128238a64ec7ec9300291759a5b6416e173d20", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "sha256": "370b7644ad5031bde2dcbb0fdc32441a741869d44d953ff315abe7ba263dbb72", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "sha256": "aa24693000c9b848f66135dde49722d19673ef1d3385736b91b5ebe298509b28", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "sha256": "0ab0d5238ba6a8d2c0b46e57ace593316c10d93759aa1328937e170c348d1a1f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "sha256": "cf3d658fb0a76fbead063bad0c9c892d7624f9b3c40e41de67cbe49169dfae58", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "sha256": "208976c7e784551983664f24330c8ba9734b4bb505e25ae6dac233c98a711052", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "sha256": "ced0d2f76cf3113b6bf7246ce1ddd90c638293a61f0c78df1b8099f3cc89a5d7", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "sha256": "03aa4d150527f6fb4ab0c84ed650e8ce63a132935ec50641e595693d378529da", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "sha256": "9a79590868ea114964ce9ca634b73213600f7800c0cd2d5b26f3e964bf496d28", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "sha256": "4182454edef4240bbc911ec01f00deae83e71a8ff878fbcceff70db3d52eaefe", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "sha256": "2553208d7552421a7e0100334a0135320dfd07d70746a04d6a75ac49c7c5514d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "sha256": "dc2453d80d4e7690f918ff3dbe9eff6663b1f837aaae3975144282126d7b1527", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "sha256": "3f0bb24ff77b733eee320eba671a0b169b1527fc8bf2dfae6c0b4cc5c363ec3f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "sha256": "4b56e03309b8339d2584a7af18609b0f5f8d041dbf08653c1a32f4a36fa9e73b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "sha256": "8c3802bb51b012a36a9012d86c020694f26ebea07492f9bdb3189248b87ec2d1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "sha256": "e8c255c31835bcf6a5193f9ff8ed91b6a94369097ed8c364168dd71d2c163e6e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "sha256": "95c5aa73aaa87b4fb944d5cd05088b1ce500d71f012d3ee1eae76d7db105b906", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "sha256": "11afcd2eebbb54385d0334e126a2ff63b90ccf820b04ed0f81ae4042273bfd66", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "sha256": "ade96cb7aa4f0a16f23920eafb683686efabee5e5aaaac023d5b11434037d181", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "sha256": "7142eefd24ec5363036bb03267e49caf7af820a8f275df4ae63afcb53471cee6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "sha256": "b689b157fd60e30ba111dbea21077086020e0e1eda146995e83b5b44cd117b43", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "sha256": "4a4e94348c97148c7332ceb32073cbf330623f507a6022ae398ccfeb6ddc2878", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "sha256": "052bdaa575e9a47e59ec21d492b5288ee289999d0121c7d0b82484f4a4f3c19f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "sha256": "969e5b48158bb841dbdc3db2388e7b13d58016649104b138e2ce307a087f4afa", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "sha256": "905dca5b90e3e301512e422eb0d5ae5631e341cb4b8dab4f5f2324ec2947a33f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "version": "10.1.0-3" }, { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime-dev10.1", + "version": "10.1.0-3" }, { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0-3" }, { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" }, { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", - "name": "rocm-smi-lib", - "version": "7.8.0.70200-43~22.04" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - } - ], - "key": "rccl_2.27.7.70200-43_22.04_amd64", - "name": "rccl", - "sha256": "bb450a46ee36578a2b171b9af891f5e94ef04e9f743f75ec16e99246ddaff878", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rccl/rccl_2.27.7.70200-43~22.04_amd64.deb" - ], - "version": "2.27.7.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", - "name": "rocm-smi-lib", - "sha256": "bfd4ef1c3382eab7685c92c218ef14593ea1bd06836bb8cc903e4100e9125d79", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70200-43~22.04_amd64.deb" - ], - "version": "7.8.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4253,253 +3884,11 @@ "key": "python3-minimal_3.10.6-1_22.04.1_amd64", "name": "python3-minimal", "version": "3.10.6-1~22.04.1" - } - ], - "key": "rocm-device-libs_1.0.0.70200-43_22.04_amd64", - "name": "rocm-device-libs", - "sha256": "f1fe8f8fea614c3f734d002f6ae688929156bcfde0f27457412a7c3074d18d51", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70200-43~22.04_amd64.deb" - ], - "version": "1.0.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "hipcc_1.1.1.70200-43_22.04_amd64", - "name": "hipcc", - "version": "1.1.1.70200-43~22.04" }, { - "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", - "name": "rocm-llvm", - "version": "22.0.0.26014.70200-43~22.04" - }, - { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", @@ -4590,362 +3979,49 @@ "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", "name": "libc-dev-bin", "version": "2.35-0ubuntu3.10" - }, - { - "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", - "name": "hsa-rocr-dev", - "version": "1.18.0.70200-43~22.04" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm2-amdgpu", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", - "name": "amdgpu-core", - "version": "1:7.2.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-amdgpu1", - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-common", - "version": "1.0.0.70200-2278374.22.04" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "file_1-5.41-3ubuntu0.1_amd64", - "name": "file", - "version": "1:5.41-3ubuntu0.1" - }, - { - "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic1", - "version": "1:5.41-3ubuntu0.1" - }, - { - "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic-mgc", - "version": "1:5.41-3ubuntu0.1" } ], - "key": "hip-dev_7.2.26015.70200-43_22.04_amd64", - "name": "hip-dev", - "sha256": "19d62d6f520b4e521fe90246d9ae872b30528e490f80c6e265f83a583302f0ad", + "key": "amdrocm-blas-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-dev10.1", + "sha256": "c7b6b25d7f1143742a7b7a2bc6819cab56a166995b60ebdc5b282d5ddc67770f", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-dev/hip-dev_7.2.26015.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-dev10.1_10.1.0-3_amd64.deb" ], - "version": "7.2.26015.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "hipcc_1.1.1.70200-43_22.04_amd64", - "name": "hipcc", - "sha256": "3c037af2114bd9058fb1420c00586c5d1c01a63d1d34500d8e61b39802ac570b", + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "sha256": "e8456d1a340533ae73e5a7cbe9aa66feea1f8cedef72971f00930167b80037a3", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipcc/hipcc_1.1.1.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64.deb" ], - "version": "1.1.1.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", - "name": "rocm-llvm", - "sha256": "88d604fe0ab4a2502f0c7e0f8d3b7b49e829ae3057971de681a68b2fdb2505d2", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26014.70200-43~22.04_amd64.deb" - ], - "version": "22.0.0.26014.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "sha256": "06faa64479f1987e4f870ff9c1ccfdf3c0c441bd3886136d6e24c11316c9f3d6", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libgcc-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "sha256": "6bf564c2bb191aefcf96758502a7ac24b2af4d644c87b3678e4b1107c5494b13", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libquadmath0_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "sha256": "becb1726d3b8a45cfd336e502fc29d7c43e024689857c670ecf75a5362732c63", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libubsan1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "sha256": "7b14e81317fce5e62d6cc2454d975d3ec4eac90710a1a536d63c9ee84a50b6f2", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libtsan0_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "sha256": "5448bba72109dbf2df6ee30463b5f06c074138006cd1d5730fb5189b93e6d563", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/gcc-11-base_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "sha256": "606e212c3cbc8fb622313fdb32f78a024b4b5b9fe54b1cc5a8f731297023b899", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/liblsan0_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "sha256": "0580e3209722dd0e6d9e57c4149a0f8f6073f975cf8b5ede26ac54289c2fc99a", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libasan6_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "sha256": "ac3c96ed50a42b4f57c410576bdf816710684e3e32e34f704a7295014565b724", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libitm1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "sha256": "072a7ee2dc00bc92a816e11ae62ad5a497c86fc1f0473c0a095f29e13003da72", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgomp1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "sha256": "5ab9c245d4c65a3986d567c54e2503251c92bd369c9f42e48a2c4b1f1835dcf5", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libstdc++-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", - "name": "hsa-rocr-dev", - "sha256": "71233ffaac8f28550c35bcf3e287a377a3c5c938a1e815e65d4155139cf4bdff", - "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70200-43~22.04_amd64.deb" - ], - "version": "1.18.0.70200-43~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-dev", - "sha256": "fec583602169ae32d3c9313dcb24c00732911234712495cbb6bb10f2194d2aee", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70200-2278374.22.04_amd64.deb" - ], - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", - "name": "libdrm-amdgpu-radeon1", - "sha256": "70049c919cff1b728107a03cd893b69106de8c00ac344e2d4019922d6bdbcd8b", - "urls": [ - "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70200-2278374.22.04_amd64.deb" - ], - "version": "1:2.4.125.70200-2278374.22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "file_1-5.41-3ubuntu0.1_amd64", - "name": "file", - "sha256": "b63b48e8b32c6f0fce5ad37d5ec3269cbbdc8feed13170aae1f416a43ebe898b", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/file_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic1", - "sha256": "344d0cc65fea4175ca1ab222c20b5b95fb1357acafea6b016add9988760829d0", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic1_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", - "name": "libmagic-mgc", - "sha256": "23fcefb8b9ee73eb74e127fc230330918a8f81e127fa54015f191cfd34726b3c", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic-mgc_5.41-3ubuntu0.1_amd64.deb" - ], - "version": "1:5.41-3ubuntu0.1" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5168,44 +4244,64 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70200-43~22.04" + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -5241,78 +4337,69 @@ "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", "name": "libc-dev-bin", "version": "2.35-0ubuntu3.10" - }, - { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" - }, - { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" - }, - { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" - }, - { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" - }, - { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" - }, - { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" - }, - { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" - }, - { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" - }, - { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" - }, - { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" } ], - "key": "rocsolver_3.32.0.70200-43_22.04_amd64", - "name": "rocsolver", - "sha256": "f18b57d3ebeba075b2b91f9e71d92da3efba73f34c5cff697cea1259b829547a", + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime-dev10.1", + "sha256": "452b3a3b31a4fa4da8d4c08976a5698aa05732b52ab88daa85cb4b4d2c84754c", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsolver/rocsolver_3.32.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime-dev10.1_10.1.0-3_amd64.deb" ], - "version": "3.32.0.70200-43~22.04" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "sha256": "351db702fd37be1fda7e5221aedf19484e99d27f2a3e7d2df5f0f531bcebcfd5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm-dev10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5335,24 +4422,79 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -5454,146 +4596,11 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "rocsolver_3.32.0.70200-43_22.04_amd64", - "name": "rocsolver", - "version": "3.32.0.70200-43~22.04" - }, - { - "key": "rocblas_5.2.0.70200-43_22.04_amd64", - "name": "rocblas", - "version": "5.2.0.70200-43~22.04" - }, - { - "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", - "name": "hip-runtime-amd", - "version": "7.2.26015.70200-43~22.04" - }, - { - "key": "libnuma1_2.0.14-3ubuntu2_amd64", - "name": "libnuma1", - "version": "2.0.14-3ubuntu2" - }, - { - "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", - "name": "rocprofiler-register", - "version": "0.6.0.70200-43~22.04" - }, - { - "key": "comgr_3.0.0.70200-43_22.04_amd64", - "name": "comgr", - "version": "3.0.0.70200-43~22.04" - }, - { - "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", - "name": "libtinfo-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", - "name": "libncurses-dev", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", - "name": "ncurses-bin", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, { "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", "name": "rpcsvc-proto", @@ -5615,76 +4622,917 @@ "version": "2.35-0ubuntu3.10" }, { - "key": "libncurses6_6.3-2ubuntu0.1_amd64", - "name": "libncurses6", + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", "version": "6.3-2ubuntu0.1" }, { - "key": "rocminfo_1.0.0.70200-43_22.04_amd64", - "name": "rocminfo", - "version": "1.0.0.70200-43~22.04" + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" }, { - "key": "pciutils_1-3.7.0-6_amd64", - "name": "pciutils", - "version": "1:3.7.0-6" + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" }, { - "key": "libpci3_1-3.7.0-6_amd64", - "name": "libpci3", - "version": "1:3.7.0-6" + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" }, { - "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", - "name": "pci.ids", - "version": "0.0~2022.01.22-1ubuntu0.1" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libudev1_249.11-0ubuntu3.16_amd64", - "name": "libudev1", - "version": "249.11-0ubuntu3.16" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "libkmod2_29-1ubuntu1_amd64", - "name": "libkmod2", - "version": "29-1ubuntu1" + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" }, { - "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", - "name": "hsa-runtime-rocr4wsl-amdgpu", - "version": "25.30.13-2281980.22.04" + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" }, { - "key": "libelf1_0.186-1ubuntu0.1_amd64", - "name": "libelf1", - "version": "0.186-1ubuntu0.1" + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" }, { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" } ], - "key": "hipsolver_3.2.0.70200-43_22.04_amd64", - "name": "hipsolver", - "sha256": "0fbff8007eada3d952db641c0653df26854854ee9e7058b4bd63202db66f2af8", + "key": "amdrocm-dnn10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1", + "sha256": "2a65817ed4bd283ff908a6789c1beaa95e1232043b9c0261edb4f7b031e1880e", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipsolver/hipsolver_3.2.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1_10.1.0-3_amd64.deb" ], - "version": "3.2.0.70200-43~22.04" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "sha256": "b945f4d2f35c53226a43ef798c036e0ea23eb4844c12a9f917fb450a41e756d4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn-host10.1", + "sha256": "199c66da1e43484d21dfe1581bcf84deec3ee82d106605058b932d8c3dc86359", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "sha256": "78e43e7ecf4a52b895fc9500e778abc0d4882a7ad5f60caff8830bb2afd64119", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "sha256": "6fa5ee1edecb61cc9434659cfece977c636da00b63d9886646848b876a60cba6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "sha256": "349a3f6c0c9cf7981d5cc2476be98e0893a5b54033a541660e084a3011efba17", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "sha256": "bffbfd01dd43330cdc2a6957b4100c64983d9e94e0fecda43dd799ce6b545534", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "sha256": "d53c6f605df4faa6cfbc4ea9a3f3e75549dbec3400ab899ab502bb8d68681d3f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "sha256": "e8e27e5fe95a8092af0c7b287e1c8b2ebb4a4099677cb274a56d09d04f6effd6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "sha256": "39609097b14134dfce5937940c0d6118f089f8963f41b88aaf82c6dbe36d8d88", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "sha256": "f9383ce30973bca37538eab93173f7a6f277189dc1f2c32285902284524b9f6c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "sha256": "daf6239c37012e979b4b31612cc27c1f6b5df16c4d0a3eb9ffdb7149f6c0c88e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "sha256": "57370823faa258ee6c28650dbaa1afe4a983f0d431c0a4f59b45c52bfd5aa776", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "sha256": "31103c9e996fb78bb9fbee801b67401fb757bf0f909d35d22ee3c555f6f2419c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "sha256": "ba17cd8597c96637216596edc2146297bf3606e32e9bac7dce06147a8d171a06", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "sha256": "25844db830121a1bb8cb3e64e1d04eb8fbda654a14c7e08b96eafce54891c789", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "sha256": "7b742fc6696895809f20edaf373a22cd50fd19f17086c56d6cb144d025b32712", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "sha256": "46658d6cf72bfc08aae94e13c7d7c34209714e700dd033b2c109203addc158b1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "sha256": "9d2165f2a2771936204ab97105a427c3b489ddc6984cebf8d3786822fbd566e9", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "sha256": "d3c75ffa1b9f79165bfcad631d009149742780114e132b3cd3e4912eb0f23632", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "sha256": "ebb48929bd2faf317aaa457096d4cefd6a9a2f8b9cbe2492bffb2e52135bdea3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "sha256": "ee57ec63c768f031899bb630b42ca780bd753ce5c2543d04aea25bd86a04bf33", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "sha256": "6ce6789fcbaaac92404402adeec89e1170ccd04cccc7340240338f506f3e0e27", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "sha256": "e33aeb4b4acd112e37a278e006fe214a19927451e63d8bfd8db7cf4fb04a03a6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "sha256": "21723b94ee22e0cc5caf839dab16c431e0b6b7f91d9a06cb22e58cc486ae7a5f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "sha256": "4b1722d77fc4e0509802a1c56466cc72a8e208ade66f83b92c32cb54ec9c357e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "sha256": "c9e57b5b1c75dcbf41dc69310adb3cdac7ef807e3119dd51a0357f2a9429f97a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", + "name": "amdrocm-fft-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5707,24 +5555,89 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -5826,6 +5739,56 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -5846,11 +5809,6 @@ "name": "readline-common", "version": "8.1.2-1" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -5861,11 +5819,6 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -5907,42 +5860,442 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "rocfft_1.0.36.70200-43_22.04_amd64", - "name": "rocfft", - "version": "1.0.36.70200-43~22.04" + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "version": "10.1.0-3" } ], - "key": "hipfft_1.0.22.70200-43_22.04_amd64", - "name": "hipfft", - "sha256": "55a791b56dbeadcc3a1f1d0d770afc71a7fbb06aed4cd190b75ee27c19c18c61", + "key": "amdrocm-fft10.1_10.1.0-3_amd64", + "name": "amdrocm-fft10.1", + "sha256": "0246b0530ab01388018cafba38ee64736933c3b5eeea0e21dcd9dafb44a5950b", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipfft/hipfft_1.0.22.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1_10.1.0-3_amd64.deb" ], - "version": "1.0.22.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "rocfft_1.0.36.70200-43_22.04_amd64", - "name": "rocfft", - "sha256": "8dbfef90c0055ef0d572451cb4a7a3380c3394485ed33cab4c116283d2fea7fe", + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx950", + "sha256": "3be1ec8ef93a0d7c4d3c463fa92a0ab0f7422f09e5c278d486e42424897704c8", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocfft/rocfft_1.0.36.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "1.0.36.70200-43~22.04" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", + "name": "amdrocm-fft-host10.1", + "sha256": "5eb075b4dd89ea9d500921959be55a469c60675f53c89e846e186da1d353c27e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx942", + "sha256": "415771e4caefb3af0e91264a6ac232ab0a1401aa68d0e9c9b836cc177c609ac2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "sha256": "c626c38512d954de19bcc298cfde38e419d4fc21d57665c10d92523c18419265", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx908", + "sha256": "3310f7fe026228c270df2b9b8100c2a64ef7aaae783a54e55ca5099fe57a718b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "sha256": "7b8cb472a25096913a4329f5dfc2e124e98621764aff2e04b79db5d7f6ce3ea8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "sha256": "eb10f46c03ffdbc1be901e2fefc16fcd3dcd36079acc17bf2a991dda010ca22e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "sha256": "04f88bce83b753730025bc352c61b16027c2a6d565f85763ce8990faf155231f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "sha256": "cd7270fd487b7abea0498a832122c11186ac234981822274af84520b8980c0c3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "sha256": "67250ae45a6190b8e0f0bea35a8019a8fd5a5da070432206a32c4ae3388f2e0e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "sha256": "f70434a361d0207188a0feff30d677164c1fdcdb0af970acbc55561b3bfc42f5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "sha256": "26b23859562047c84e8eb4ff7387bcab69b475eeb8655e577b0c95e436e6c2f4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "sha256": "0cd8464f08f8b6b08ed10773352a34d1d1efea4de3c039a36e6f3bb4fdfb0f9d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "sha256": "ec4b9439df683ee782241ec4551daa4e0ec4e4ae4a8c2b92f1bd41285cad596e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "sha256": "2ed4b2fbabb310fdb7fdf8ed01c9ca1f0f37fe97b5e3027451737be4815492a8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "sha256": "df323baa9cfb3a2c2dd81bd81510021db4780eb074717b22abb9dbd80d992dc0", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "sha256": "6eb45f4a815e10d2119370e0ed68fe44bc34c2999c3b0a799d15d23d96f17ed8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "sha256": "1da409970e160027c8e5fbf9d446394fa9108b92163ba69796b7d7f598f65d18", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "sha256": "58351fe6a54cc374a3d94a61416c7c4079303e2b38e974d54381bd4f5b53f83e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "sha256": "98bb8653582f9840ba723b0344204be8f6c55c758e756acec6ee372e9233119d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "sha256": "e4b0fc5dab2eaac470a54811d19bef3b9c1f903e94d5dfa56a4f3f05b8d5977d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "sha256": "998eae13396245c496da21504fff6832654b50323cc40cb694060a79f943a51c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "sha256": "e321ced70752e27d4c8b70eaf3b8590b9ce6019d64896cb5cfca75d4ad26bf89", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "sha256": "b8910ca5eea38a9b9e68d3752637e214ded07e84b4ac83da39f15157cc56bf64", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "sha256": "f6cc9a8657250ba79d739ca65bc8cb13491629695b9c486060652ea9fb508656", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "sha256": "378078f09766726732a4f192397b6f1d9fee346b68efd519b58238f9c41f895a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [ { - "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt", - "version": "1.2.1.70200-43~22.04" + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "version": "10.1.0-3" }, { - "key": "rocm-core_7.2.0.70200-43_22.04_amd64", - "name": "rocm-core", - "version": "7.2.0.70200-43~22.04" + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5965,24 +6318,89 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" }, { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" }, { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" }, { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -6084,6 +6502,56 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -6104,11 +6572,6 @@ "name": "readline-common", "version": "8.1.2-1" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -6119,11 +6582,6 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -6165,34 +6623,1844 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "roctracer_4.1.70200.70200-43_22.04_amd64", - "name": "roctracer", - "version": "4.1.70200.70200-43~22.04" + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", + "name": "amdrocm-amdsmi10.1", + "version": "10.1.0-3" }, { - "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", - "name": "hipblas-common-dev", - "version": "1.4.0.70200-43~22.04" + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "version": "10.1.0-3" } ], - "key": "hipblaslt-dev_1.2.1.70200-43_22.04_amd64", - "name": "hipblaslt-dev", - "sha256": "7bb01f62e51be15495ea4a124db7b6c4e600d27d401ce636fb9b3daaf6b4b958", + "key": "amdrocm-rccl10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1", + "sha256": "18b5e340191a729524ede9ec9d1810f3673d035b4940743307189e1b4a96d293", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.1.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1_10.1.0-3_amd64.deb" ], - "version": "1.2.1.70200-43~22.04" + "version": "10.1.0-3" }, { "arch": "amd64", "dependencies": [], - "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", - "name": "hipblas-common-dev", - "sha256": "1eed050f8367b58d697db452c3928780ae408ee055694f1676326d54f4caa739", + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "sha256": "d9a77415ca9392af357d188df8e805f86c8b0f7574705133ee919b88fec66f4d", "urls": [ - "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70200-43~22.04_amd64.deb" + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx950_10.1.0-3_amd64.deb" ], - "version": "1.4.0.70200-43~22.04" + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl-host10.1", + "sha256": "b38172c360fc8fdf89d038edfa3cb5c3cb874804f089a4c9ba91c384be7c9707", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "sha256": "c0c4be063b49364858d76af509cbf88e4642146489a1fcde2088f66a2e1f31a9", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "sha256": "ffcc6bce6b2271b3c6cd1c5d7b568fe2773a0022a2ba4d24c177f7df5d6c5225", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "sha256": "2eebb6a869970c182dd64105da18f956801ed7a048693f47f99a05a087b01f59", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "sha256": "941da91fc0f827732d0c4e0010403723643a27dd3ca2494270aa97d0a8a38f35", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "sha256": "4c589250c3c587d4865551b15f32adb46cfbb5aaa07e147eb82b3d6634cff213", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "sha256": "33086fcdf5aa8e353c90ac38effddca41cb3d8ae88272333df2c580b08826abe", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "sha256": "4d7b24fc17351d69c63028867b5bdf4b9015e563175c868a814c8ca34b7bbcce", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "sha256": "935b928d8f2901861021d3f9cb2c43953d309975c4e5803b4ac496d6bd4ee007", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "sha256": "4d418a9fb372ec15bd9e42faa02f9bf945adc5de2853cb77db54cb658286c67c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "sha256": "ceddf293510c927d2a5a8fee30cc76fce6cd0d3cc6977422396514bedbcc597e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "sha256": "8fb0c4dfa528a82c872df49b0feeb7205e9664ce06d9760d0fa85a1892d97f79", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "sha256": "174385d65d68e5c5aa7c51f5cfcf90fae9005c37944f4515883f370760830b74", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "sha256": "c3e4727c061c15aa67b89b61539d58be182dca850df13defe1790b7b5b7ffd4d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "sha256": "521d5e2bc7c7d8f4d941044472fd9f800d5f172d4c9a3c015b035e2454c2dba1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "sha256": "198345dc05c10db202080744337fee857c367786a315004e662dc24340cff787", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "sha256": "91761516a93977fcc70cb771e5ad2e7f958f8147905ca86adf592e5f7d79dab5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "sha256": "076013938bd6ef83e31d5c4f444d7fb136d0733a4edfdfccf05d8108776c3388", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "sha256": "473c3be9bf38e2f30dac5f6019a521c3aad2b96e10e28389f542c799141b8f09", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "sha256": "842068f0334f6c3d4977394c9ee5098f8a8254012206b5f66f5b2aaa0e510ce1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "sha256": "09c782685d0d6a93e767a580dc5e35183cf9f742859953271d8abaa977a74c6e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-solver10.1_10.1.0-3_amd64", + "name": "amdrocm-solver10.1", + "sha256": "66f914aa374d13a700da9006af9dcd4aeae20c02a50b4807c45d3ed990da0711", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-sparse10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1", + "sha256": "71a68bcbb89fdac799535916b602a7f456269a24e2e40d0151abd003c7ad1b58", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index b9b96902..c9116561 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,10 +4,10 @@ version: 1 sources: - - channel: jammy main - url: https://repo.radeon.com/rocm/apt/7.2 - - channel: jammy main - url: https://repo.radeon.com/graphics/7.2/ubuntu + - channel: llvm-toolchain-jammy-22 main + url: https://apt.llvm.org/jammy + - channel: stable main + url: https://stable.repo.amd.com/rocm/core/packages/ubuntu2204 - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -18,25 +18,19 @@ sources: archs: - "amd64" -# readelf -d libpjrt_rocm.so | grep NEEDED packages: - # - "rocm-smi-lib" - - "libdrm-amdgpu-common" - # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu - - "hsa-rocr (<< 20)" - - "hsa-amd-aqlprofile" - - "comgr" - - "rocprofiler-register" - - "rocprofiler-sdk" - - "miopen-hip" - - "rccl" - - "rocm-device-libs" - - "hip-dev" - - "rocblas" - - "rocsolver" - - "hipsolver" - - "hipfft" - - "roctracer" - - "hipblaslt" - - "hipblaslt-dev" - - "hip-runtime-amd" + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-jammy-22. + - "libatomic1" + - "libdrm-common" + - "amdrocm-amdsmi10.1" + - "amdrocm-base10.1" + - "amdrocm-blas10.1" + - "amdrocm-blas-dev10.1" + - "amdrocm-dnn10.1" + - "amdrocm-fft10.1" + - "amdrocm-llvm10.1" + - "amdrocm-profiler-base10.1" + - "amdrocm-rccl10.1" + - "amdrocm-runtime-dev10.1" + - "amdrocm-solver10.1" + - "amdrocm-sparse10.1" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 78171a3b..41b01b60 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -1,126 +1,236 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") """ -_ROCM_STRIP_PREFIX = "opt/rocm-7.2.0" +_ROCM_VERSION = "10.1" +_ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" + +def _rocm_package_name(name): + return name + _ROCM_VERSION + +def _rocm_repo_name(package_name): + return package_name.replace(_ROCM_VERSION, "") _UBUNTU_PACKAGES = { - "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), - "libelf1": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libelf1", - src = "usr/lib/x86_64-linux-gnu/libelf.so.1", - set_rpath = "$ORIGIN", - ), - ]), - "libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), - "libnuma1": packages.filegroup(name = "libnuma1", srcs = ["usr/lib/x86_64-linux-gnu/libnuma.so.1"]), - "libzstd1": packages.filegroup(name = "libzstd1", srcs = ["usr/lib/x86_64-linux-gnu/libzstd.so.1"]), - "libdrm-amdgpu-amdgpu1": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libdrm-amdgpu-amdgpu1", - src = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", - add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN", - rename_dynamic_symbols = { - "fopen64": "zmlxrocm_fopen64", - }, - ), - ]), - "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), - "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + "llvm-libunwind1": packages.filegroup(name = "libunwind", srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"]), + "libatomic1": packages.filegroup(name = "libatomic1", srcs = ["usr/lib/x86_64-linux-gnu/libatomic.so.1"]), + "libdrm-common": packages.filegroup(name = "amdgpu_ids", srcs = ["usr/share/libdrm/amdgpu.ids"]), } -_ROCM_PACKAGES = { - "rocm-core": packages.filegroup(name = "rocm-core", srcs = ["lib/librocm-core.so.1"]), - "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.1"]), - "rocprofiler-sdk": "\n".join([ - packages.filegroup(name = "rocprofiler-sdk", srcs = [ - "lib/librocprofiler-sdk.so.1", - "lib/librocprofiler-sdk-attach.so.1", +def _rocm_dlopen_patchelf(name, src): + return "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = name, + src = src, + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + "pthread_attr_setstacksize": "zmlxrocm_pthread_attr_setstacksize", + }, + ), + ]) + +def _glob_filegroup(name, srcs): + return """filegroup( + name = "{name}", + srcs = glob({srcs}), +)""".format(name = name, srcs = repr(srcs)) + +def _family_runfile_package_names(loaded_packages, family): + prefix = "amdrocm-{}{}-gfx".format(family, _ROCM_VERSION) + return sorted([ + pkg_name + for pkg_name in loaded_packages.keys() + if pkg_name.startswith(prefix) + ]) + +def _family_runfile_labels(loaded_packages, family): + return [ + "@{}//:runfiles".format(_rocm_repo_name(pkg_name)) + for pkg_name in _family_runfile_package_names(loaded_packages, family) + ] + +def _rocm_runfile_build_files(loaded_packages): + rocm_packages = {} + + for pkg_name in _family_runfile_package_names(loaded_packages, "blas"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", + "lib/hipblaslt/library/**", + "lib/rocblas/library/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", + "lib/libMIOpenCKGroupedConv_*.so", + "share/miopen/db/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "fft"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "rccl"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + return rocm_packages + +def _rocm_base_build_files(loaded_packages): + return { + _rocm_package_name("amdrocm-amdsmi"): "\n\n".join([ + packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), + packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.27"]), ]), - ]), - "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), - "rocprofiler-sdk-roctx": packages.filegroup(name = "rocprofiler-sdk-roctx", srcs = ["lib/librocprofiler-sdk-roctx.so.1"]), - "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), - "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), - "comgr": packages.filegroup(name = "amd_comgr", srcs = ["lib/libamd_comgr.so.3"]), - "rocprofiler-register": packages.filegroup(name = "rocprofiler-register", srcs = ["lib/librocprofiler-register.so.0"]), - "miopen-hip": "\n".join([ - packages.filegroup(name = "MIOpen", srcs = ["lib/libMIOpen.so.1"]), - """filegroup(name = "runfiles", srcs = glob(["share/miopen/**"]))""", - ]), - "rccl": packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), - "rocm-device-libs": """filegroup(name = "runfiles", srcs = glob(["amdgcn/**"]))""", - "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", - "rocblas": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "rocblas", - src = "lib/librocblas.so.5", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - """filegroup( - name = "runfiles", - srcs = glob(["lib/rocblas/library/**"]), - ) - """ - ]), - "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), - "rocsolver": packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), - "roctracer": "\n".join([ - packages.filegroup(name = "roctracer", srcs = ["lib/libroctracer64.so.4"]), - packages.filegroup(name = "roctx", srcs = ["lib/libroctx64.so.4"]), - ]), - "hipblaslt": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "hipblaslt", - src = "lib/libhipblaslt.so.1", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.patchelf( - name = "rocroller", - src = "lib/librocroller.so.1", - set_rpath = "$ORIGIN", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - """filegroup( - name = "runfiles", - srcs = glob(["lib/hipblaslt/library/**"]), - ) - """ - ]), - "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), - "hip-runtime-amd": "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "amdhip", - src = "lib/libamdhip64.so.7", - add_needed = ["libzmlxrocm.so.0"], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - ), - packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7", "lib/libhiprtc-builtins.so.7"]), - ]), - "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), -} + _rocm_package_name("amdrocm-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "rocm-core", src = "lib/librocm-core.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-register", src = "lib/librocprofiler-register.so.0"), + ]), + _rocm_package_name("amdrocm-blas-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-blas-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipblas", src = "lib/libhipblas.so.3"), + packages.filegroup(name = "hipblaslt", srcs = [ + "lib/libhipblaslt.so.1", + "lib/liborigami.so.1", + ]), + packages.filegroup(name = "rocblas", srcs = ["lib/librocblas.so.5"]), + _rocm_dlopen_patchelf(name = "rocroller", src = "lib/librocroller.so.1"), + _glob_filegroup("hipblaslt_support", [ + "lib/hipdnn_plugins/engines/libhipblaslt_plugin.so", + "share/hipblaslt/**", + ]), + ]), + _rocm_package_name("amdrocm-blas"): "\n\n".join([ + packages.filegroup(name = "hipblaslt", srcs = [ + "@amdrocm-blas-host//:hipblaslt", + "@amdrocm-blas-host//:rocroller", + ]), + packages.filegroup(name = "hipblaslt_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas") + [ + "@amdrocm-blas-host//:hipblaslt_support", + ]), + packages.filegroup(name = "rocblas", srcs = [ + "@amdrocm-blas-host//:hipblas", + "@amdrocm-blas-host//:rocblas", + ]), + packages.filegroup(name = "rocblas_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas")), + ]), + _rocm_package_name("amdrocm-dnn-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "MIOpen", src = "lib/libMIOpen.so.1"), + _rocm_dlopen_patchelf(name = "hipdnn_backend", src = "lib/libhipdnn_backend.so"), + _glob_filegroup("runfiles", [ + "lib/hipdnn_plugins/**", + "share/miopen/**", + ]), + ]), + _rocm_package_name("amdrocm-dnn"): packages.filegroup(name = "MIOpen", srcs = [ + "@amdrocm-dnn-host//:MIOpen", + "@amdrocm-dnn-host//:hipdnn_backend", + "@amdrocm-dnn-host//:runfiles", + ] + _family_runfile_labels(loaded_packages, "dnn")), + _rocm_package_name("amdrocm-fft-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipfft", src = "lib/libhipfft.so.0"), + _rocm_dlopen_patchelf(name = "hipfftw", src = "lib/libhipfftw.so.0"), + packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), + ]), + _rocm_package_name("amdrocm-fft"): "\n\n".join([ + packages.filegroup(name = "hipfft", srcs = [ + "@amdrocm-fft-host//:hipfft", + "@amdrocm-fft-host//:hipfftw", + ]), + packages.filegroup(name = "rocfft", srcs = [ + "@amdrocm-fft-host//:rocfft", + ] + _family_runfile_labels(loaded_packages, "fft")), + ]), + _rocm_package_name("amdrocm-llvm"): "\n\n".join([ + packages.filegroup(name = "llvm_libs", srcs = [ + "lib/llvm/lib/libLLVM.so.24.0git", + "lib/llvm/lib/libclang-cpp.so.24.0git", + ]), + _glob_filegroup("rocm_device_libs_runfiles", ["lib/llvm/amdgcn/**"]), + ]), + _rocm_package_name("amdrocm-math-common"): _glob_filegroup("host_math_libs", ["lib/host-math/lib/*.so*"]), + _rocm_package_name("amdrocm-profiler-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hsa-amd-aqlprofile", src = "lib/libhsa-amd-aqlprofile64.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-sdk_so", src = "lib/librocprofiler-sdk.so.1"), + packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), + packages.filegroup(name = "rocprofiler-sdk-rocattach", srcs = ["lib/librocprofiler-sdk-rocattach.so.1"]), + _rocm_dlopen_patchelf(name = "roctracer", src = "lib/libroctracer64.so.4"), + _rocm_dlopen_patchelf(name = "roctx", src = "lib/libroctx64.so.4"), + packages.cc_library_hdrs_glob( + name = "roctx_headers", + hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], + includes = ["include"], + ), + packages.filegroup(name = "rocprofiler-sdk", srcs = [ + ":rocprofiler-sdk_so", + ":rocprofiler-sdk-attach", + ":rocprofiler-sdk-rocattach", + ]), + _glob_filegroup("rocprofv3_tool", [ + "bin/rocprof*", + "bin/rocpd*", + "libexec/rocprofiler-sdk/**", + "share/rocprofiler-sdk/**", + ]), + ]), + _rocm_package_name("amdrocm-rccl-host"): packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), + _rocm_package_name("amdrocm-rccl"): packages.filegroup(name = "rccl", srcs = [ + "@amdrocm-rccl-host//:rccl", + ] + _family_runfile_labels(loaded_packages, "rccl")), + _rocm_package_name("amdrocm-runtime"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "amd_comgr", src = "lib/libamd_comgr.so.3"), + _rocm_dlopen_patchelf(name = "amdhip_so", src = "lib/libamdhip64.so.7"), + _rocm_dlopen_patchelf(name = "hiprtc_so", src = "lib/libhiprtc.so.7"), + _rocm_dlopen_patchelf(name = "hsa-runtime", src = "lib/libhsa-runtime64.so.1"), + packages.filegroup(name = "amdhip", srcs = [ + ":amdhip_so", + "lib/librocm_kpack.so.0", + ]), + packages.filegroup(name = "hiprtc", srcs = [ + ":hiprtc_so", + "lib/libhiprtc-builtins.so.7", + ]), + ]), + _rocm_package_name("amdrocm-runtime-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-solver-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsolver", src = "lib/libhipsolver.so.1"), + packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), + ]), + _rocm_package_name("amdrocm-solver"): "\n\n".join([ + packages.filegroup(name = "hipsolver", srcs = ["@amdrocm-solver-host//:hipsolver"]), + packages.filegroup(name = "rocsolver", srcs = [ + "@amdrocm-solver-host//:rocsolver", + "@amdrocm-math-common//:host_math_libs", + ]), + ]), + _rocm_package_name("amdrocm-sparse-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsparse", src = "lib/libhipsparse.so.4"), + packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), + ]), + _rocm_package_name("amdrocm-sparse"): packages.filegroup(name = "rocsparse", srcs = [ + "@amdrocm-sparse-host//:hipsparse", + "@amdrocm-sparse-host//:rocsparse", + ]), + _rocm_package_name("amdrocm-sysdeps"): _glob_filegroup("system_libs", [ + "lib/rocm_sysdeps/lib/*.so*", + "lib/rocm_sysdeps/share/**", + ]), + } + +def _rocm_build_files(loaded_packages): + rocm_packages = _rocm_base_build_files(loaded_packages) + rocm_packages.update(_rocm_runfile_build_files(loaded_packages)) + return rocm_packages def _rocm_impl(mctx): loaded_packages = packages.read(mctx, [ @@ -128,7 +238,7 @@ def _rocm_impl(mctx): ]) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -136,26 +246,50 @@ def _rocm_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) - for pkg_name, build_file_content in _ROCM_PACKAGES.items(): - pkg = loaded_packages[pkg_name] + for pkg_name, build_file_content in _rocm_build_files(loaded_packages).items(): + pkg = loaded_packages[pkg_name]["amd64"] http_deb_archive( - name = pkg_name, + name = _rocm_repo_name(pkg_name), urls = pkg["urls"], sha256 = pkg["sha256"], strip_prefix = _ROCM_STRIP_PREFIX, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) + http_file( + name = "libdrm_mesa_amdgpu_ids", + url = "https://gitlab.freedesktop.org/mesa/libdrm/-/raw/979f607906ad64f629967ac1f3ba3590e756442c/data/amdgpu.ids?inline=false", + sha256 = "ffd2a8f1bfa755f4d90f537b4969fc4676f116e5af051ce2f18ef93a96d8beb6", + downloaded_file_path = "amdgpu.ids", + ) + http_archive( - name = "libpjrt_rocm", - build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v16.0.0/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "6a70cf6a5a8330809f173072dd5fae5332c3ec8fba80acb9b77ddd97bda6572b", + name = "libzml_rocm", + build_file = "libzml_rocm.BUILD.bazel", + url = _PLUGIN_ROCM_URL, + sha256 = _PLUGIN_ROCM_SHA256, ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_rocm", "hipblaslt", "rocblas"], + root_module_direct_deps = [ + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libzml_rocm", + ], root_module_direct_dev_deps = [], ) diff --git a/platforms/rocm/rocm.h b/platforms/rocm/rocm.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/rocm/rocm.h @@ -0,0 +1 @@ + #include diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index 79053208..ee31bd83 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -5,13 +5,14 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/rocm"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_ROCM"); + return platforms_options.rocm_enabled; } fn hasRocmDevices(io: std.Io) bool { @@ -24,10 +25,23 @@ fn hasRocmDevices(io: std.Io) bool { fn setupRocmEnv(rocm_data_dir: []const u8) !void { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated + + // Use one HIP graph stream to reduce cross-stream synchronization overhead. + _ = c.setenv("DEBUG_HIP_FORCE_GRAPH_QUEUES", "1", 0); + // Share one hardware queue per device so auxiliary barriers stay on the kernel queue. + _ = c.setenv("GPU_MAX_HW_QUEUES", "1", 0); + // Disable rocprofiler queue interposition to avoid its dispatch overhead outside profiling. + _ = c.setenv("ROCPROFILER_QUEUE_INTERPOSITION", "0", 0); + + if (c.getenv("ROCPROFILER_LOG_LEVEL") == null) { + // rocprofiler defaults to the `warning` log level, which can produce unnecessary log output. + // Set the log level to `fatal` to suppress all but fatal error messages. + _ = c.setenv("ROCPROFILER_LOG_LEVEL", "error", 0); + } } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } @@ -41,7 +55,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_rocm/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for ROCm runtime", .{}); return error.FileNotFound; }; @@ -59,7 +73,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { // executing the destructor. Accessing this variable results in a segmentation fault... return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); break :blk .loadFrom(lib_path); }; } diff --git a/platforms/rocm/zmlxrocm.zig b/platforms/rocm/zmlxrocm.zig index e3024de6..069c08bb 100644 --- a/platforms/rocm/zmlxrocm.zig +++ b/platforms/rocm/zmlxrocm.zig @@ -4,22 +4,11 @@ const stdx = @import("stdx"); pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "librocm-core.so", "librocm-core.so.1" }, - .{ "librocm_smi64.so", "librocm_smi64.so.1" }, - .{ "libhsa-runtime64.so", "libhsa-runtime64.so.1" }, - .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, - .{ "libamd_comgr.so", "libamd_comgr.so.3" }, - .{ "librocprofiler-register.so", "librocprofiler-register.so.0" }, - .{ "libMIOpen.so", "libMIOpen.so.1" }, - .{ "libMIOpen.so.1", "libMIOpen.so.1" }, - .{ "librccl.so", "librccl.so.1" }, - .{ "librocblas.so.5", "librocblas.so.5" }, - .{ "librocblas.so", "librocblas.so.5" }, - .{ "libroctracer64.so", "libroctracer64.so.4" }, - .{ "libroctx64.so", "libroctx64.so.4" }, - .{ "libhipblaslt.so", "libhipblaslt.so.1" }, .{ "libamdhip64.so", "libamdhip64.so.7" }, - .{ "libhiprtc.so", "libhiprtc.so.7" }, + .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, + .{ "librocprofiler-sdk-attach.so", "librocprofiler-sdk-attach.so.1" }, + .{ "librocprofiler-sdk.so", "librocprofiler-sdk.so.1" }, + .{ "libroctx64.so", "libroctx64.so.4" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; @@ -50,3 +39,29 @@ pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std return std.c.fopen64(new_pathname, mode); } + +extern fn pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int; + +// ROCclr asks for small helper-thread stacks: CQ_THREAD_STACK_SIZE is 256 KiB, +// then ROCclr adds the pthread guard size before calling pthread_attr_setstacksize. +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/utils/flags.hpp#L25 +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/os/os_posix.cpp#L378-L385 +// glibc 2.28 accepts stack sizes >= PTHREAD_STACK_MIN at attribute-set time, +// but pthread_create can still reject stacks that do not leave room for the +// guard, static TLS, and MINIMAL_REST_STACK: +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/pthread_attr_setstacksize.c#L35-L38 +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/allocatestack.c#L532-L544 +// The 1 MiB floor below is a local compatibility margin for that glibc path; +// it is intentionally larger than ROCclr's request and not a ROCm ABI value. +pub export fn zmlxrocm_pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int { + const min_stack_size = 1024 * 1024; + const adjusted_stacksize = blk: { + if (stacksize < min_stack_size) { + break :blk min_stack_size; + } else { + break :blk stacksize; + } + }; + + return pthread_attr_setstacksize(attr, adjusted_stacksize); +} diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index af25cd2f..1aab4031 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -2,30 +2,38 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( - name = "empty", - defines = ["ZML_RUNTIME_TPU_DISABLED"], -) - -cc_library( - name = "libpjrt_tpu", - defines = ["ZML_RUNTIME_TPU"], - deps = ["@libpjrt_tpu"], + name = "libzml_tpu", + tags = ["manual"], + deps = ["@libzml_tpu"], ) zig_library( name = "tpu", import_name = "platforms/tpu", main = "tpu.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ + "//bazel", "//pjrt", + "//platforms:options", + "//stdx", ] + select({ "//platforms:tpu.enabled": [ - ":libpjrt_tpu", - "//bazel", - "//stdx", - "@rules_zig//zig/runfiles", + ":libzml_tpu", ], - "//conditions:default": [":empty"], + "//conditions:default": [], }), ) + +zig_library( + name = "ragged_paged", + import_name = "platforms/tpu/ragged_paged", + main = "ragged_paged.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/mosaic_tpu:builder", + "//mlir", + ], +) diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel deleted file mode 100644 index 767cd004..00000000 --- a/platforms/tpu/libpjrt_tpu.BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -copy_file( - name = "libpjrt_tpu.so", - src = "libtpu/libtpu.so", - out = "lib/libpjrt_tpu.so", -) - -copy_to_directory( - name = "sandbox", - srcs = [ - "lib/libpjrt_tpu.so", - ], -) - -cc_library( - name = "libpjrt_tpu", - data = [":sandbox"], - visibility = ["@zml//platforms/tpu:__subpackages__"], -) diff --git a/platforms/tpu/libzml_tpu.BUILD.bazel b/platforms/tpu/libzml_tpu.BUILD.bazel new file mode 100644 index 00000000..1186510c --- /dev/null +++ b/platforms/tpu/libzml_tpu.BUILD.bazel @@ -0,0 +1,35 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +copy_file( + name = "libzml_tpu.so", + src = "libtpu/libtpu.so", + out = "lib/libzml_tpu.so", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/libzml_tpu.so", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/include/**", + "**/_solib_*/**", + ], + hardlink = "auto", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "**/lib**", + ], + replace_prefixes = { + "lib/x86_64-linux-gnu": "lib", + }, +) + +cc_library( + name = "libzml_tpu", + data = [":sandbox"], + visibility = ["@zml//platforms/tpu:__subpackages__"], +) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig new file mode 100644 index 00000000..6a0fdfae --- /dev/null +++ b/platforms/tpu/ragged_paged.zig @@ -0,0 +1,715 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mtt = @import("dsl/mosaic_tpu/builder"); + +pub const Cfg = struct { + // Shapes that vary at compile-of-kernel time. + num_q_tokens: i64, + num_q_heads: i64, + num_kv_heads: i64, + head_dim: i64, + total_num_pages: i64, + page_size: i64, + max_num_seqs: i64, + pages_per_seq: i64, + + // Block-level tuning knobs. + num_kv_pages_per_block: i64, + num_queries_per_block: i64, + + // Element types. + q_dtype: mtt.DType = .bf16, + kv_dtype: mtt.DType = .bf16, + + // Numeric knobs. + sm_scale: f32, + mask_value: f32 = -2.38197633e+38, + /// Pre-mask softcap: `qk = soft_cap * tanh(qk / soft_cap)`. + soft_cap: ?f32 = null, + /// K dequant scale applied right after `stridedLoadKv` (mirrors Pallas's + /// `k.astype(f32) * k_scale → k.astype(q_dtype)`). + k_scale: ?f32 = null, + v_scale: ?f32 = null, + + /// Whether to mask keys beyond the current query row. + is_causal: bool = true, + + /// Sliding-window mask added on top of the validity/causal mask: any + /// `(row, col)` with `row - sliding_window >= col` is masked out. Mirrors + /// `kernel.py:391-393` in causal mode. + sliding_window: ?i64 = null, + vmem_limit_bytes: ?i64 = null, + + // Derived shape helpers — keeps the body readable. + pub fn numCombinedKvHeads(self: Cfg) i64 { + return 2 * self.num_kv_heads; + } + pub fn numQHeadsPerKvHead(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.num_kv_heads); + } + pub fn numQHeadsPerBlk(self: Cfg) i64 { + return self.num_q_heads; + } + pub fn numCombinedKvHeadsPerBlk(self: Cfg) i64 { + return self.numCombinedKvHeads(); + } + pub fn numKvHeadsPerBlk(self: Cfg) i64 { + return @divExact(self.numCombinedKvHeadsPerBlk(), 2); + } + pub fn numQPerBlk(self: Cfg) i64 { + return self.num_queries_per_block; + } + pub fn numKvPerBlk(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size; + } + pub fn numHeadsBlks(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.numQHeadsPerBlk()); + } + pub fn numQBlks(self: Cfg) i64 { + return @divExact(self.num_q_tokens, self.numQPerBlk()); + } + pub fn flashQRows(self: Cfg) i64 { + return self.numQPerBlk() * self.numQHeadsPerKvHead(); + } + pub fn flashKRows(self: Cfg) i64 { + return self.numKvPerBlk(); + } + + /// Pallas's `get_dtype_packing(dtype) = 32 // bitwidth(dtype)`. Mirrors + /// `kernel.py:get_dtype_packing` exactly. + pub fn kvPackingFactor(self: Cfg) i64 { + return switch (self.kv_dtype) { + .i32, .f32 => 1, + .i16, .f16, .bf16 => 2, + .i8, .f8e4m3fn, .f8e5m2 => 4, + else => std.debug.panic("ragged_paged: unsupported KV dtype {s}", .{@tagName(self.kv_dtype)}), + }; + } + /// Same packing rule applied to `q_dtype` — used by `fold_on_2nd_minor` + /// to decide whether to cast q to f32 before the trailing-dim reshape. + pub fn qPackingFactor(self: Cfg) i64 { + return switch (self.q_dtype) { + .f32 => 1, + .f16, .bf16 => 2, + else => std.debug.panic("ragged_paged: unsupported Q dtype {s}", .{@tagName(self.q_dtype)}), + }; + } + /// `kv_load_step = max(1, kv_packing // 2)` — number of (k, v) pairs the + /// caller unpacks from a single `b` strided-load (Pallas's outer-loop step). + pub fn kvLoadStep(self: Cfg) i64 { + const p = self.kvPackingFactor(); + return @max(@as(i64, 1), @divFloor(p, 2)); + } + pub fn kvFlatRows(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size * self.numCombinedKvHeadsPerBlk(); + } + pub fn kvFlatI32Rows(self: Cfg) i64 { + return @divExact(self.kvFlatRows(), self.kvPackingFactor()); + } +}; + +fn maskedStoreLM( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + kv_head_idx_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, + group: i64, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const iota_div = k.divFloor(iota, @as(i32, @intCast(group))); + const ge = k.cmpi(.sge, iota_div, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota_div, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + const slot_shape = [_]i64{ 1, shape[0], shape[1] }; + const val_3d = k.shapeCast(val, &slot_shape); + const mask_3d = k.shapeCast(mask, &slot_shape); + k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.constIndex(0), k.constIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); +} + +/// `pltpu.store(acc_ref, val, mask=…)` — group=1 so no floor-div. +fn maskedStoreAcc( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + q_head_offset_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const ge = k.cmpi(.sge, iota, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + k.refStoreOpts(ref, val, &.{ k.constIndex(0), q_head_offset_idx, k.constIndex(0) }, .{ .mask = mask, .strides = &.{} }); +} + +/// `MultiPageAsyncCopyDescriptor.__init__` — `tpu.memref_slice → squeeze +/// → enqueue_dma` chain that Pallas emits for one async page-copy. +fn createKvAsyncCopyDescriptors( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, // i32 + seq_idx: mtt.Value, // i32 + kv_blk_idx: mtt.Value, // i32 — the block-relative page index + buf_idx: mtt.Value, // i32 +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), + ); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); + + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.enqueueDma(src, dst, sem, .{}); +} + +fn waitKvDma( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, + seq_idx: mtt.Value, + kv_blk_idx: mtt.Value, + buf_idx: mtt.Value, +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), + ); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.waitDma2(sem, src, dst, .{}); +} + +const KvPair = struct { k: mtt.Value, v: mtt.Value }; + +/// Mirrors Pallas's `b_ref[b_start::b_step, :]` lowering: +/// - `b_step == 1` → `vector.load` (the slice is contiguous). +/// - `last_dim ≤ 128` and `b_step > 1` → `tpu.strided_load`. +/// - `last_dim > 128` → per-row `vector.load` + `tpu.concatenate` +/// (`jnp.stack([ref[start + i*step, :] for i in range(num_rows)])`). +fn loadStridedOrStack( + k: *mtt.Builder, + ref: mtt.Value, + start_row: i64, + stride: i32, + out_rows: i64, + out_last_dim: i64, +) mtt.Value { + if (out_last_dim <= 128) { + if (stride == 1) { + return k.vectorLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ out_rows, out_last_dim }); + } + return k.stridedLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); + } + const rows_count: usize = @intCast(out_rows); + const rows = k.arena.allocator().alloc(mtt.Value, rows_count) catch @panic("loadStridedOrStack OOM"); + for (0..rows_count) |i| { + const row_idx: i64 = start_row + @as(i64, @intCast(i)) * @as(i64, stride); + rows[i] = k.vectorLoadShape(ref, &.{ k.constIndex(row_idx), k.constIndex(0) }, &.{ 1, out_last_dim }); + } + return k.concatenate(rows, 0); +} + +/// Pallas's `broadcast_to_shape(arr, shape)` — emit `tpu.concatenate` of +/// `shape[1] / arr.shape[1]` copies when expanding axis 1 (e.g., +/// `` → `` for `head_dim > 128`). +fn broadcastToShape(k: *mtt.Builder, arr: mtt.Value, shape: []const i64) mtt.Value { + const arr_shape = arr.shape(); + std.debug.assert(arr_shape.len == 2 and shape.len == 2); + if (arr_shape.get(1) == shape[1]) return arr; + std.debug.assert(arr_shape.get(0) == shape[0]); + std.debug.assert(@mod(shape[1], arr_shape.get(1)) == 0); + const repeats: usize = @intCast(@divExact(shape[1], arr_shape.get(1))); + const sources = k.arena.allocator().alloc(mtt.Value, repeats) catch @panic("broadcastToShape OOM"); + for (sources) |*s| s.* = arr; + return k.concatenate(sources, 1); +} + +/// `strided_load_kv(ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk)` +/// — full Pallas parity. +/// +/// - `packing == 1` (i32/f32): two strided loads in original dtype. +/// - `packing == 2` bf16: special-case shli + and + tpu.bitcast(f32) + truncf. +/// - `packing == 2` f16/i16, `packing == 4` (i8/u8/f8): generic shrui + trunci +/// + tpu.bitcast unpack producing `kv_load_step` (k, v) pairs. +fn stridedLoadKv( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + buf_idx: mtt.Value, + kv_head_chunk_idx: i64, +) []const KvPair { + const packing = cfg.kvPackingFactor(); + const start: i64 = kv_head_chunk_idx * 2; + const step: i32 = @intCast(cfg.numCombinedKvHeadsPerBlk()); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const flat = k.memRefReshape(dst1, &.{ cfg.kvFlatRows(), cfg.head_dim }); + + if (packing == 1) { + const k_v = loadStridedOrStack(k, flat, start, step, cfg.flashKRows(), cfg.head_dim); + const v_v = loadStridedOrStack(k, flat, start + 1, step, cfg.flashKRows(), cfg.head_dim); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + std.debug.assert(packing == 2 or packing == 4 or packing == 8); + std.debug.assert(@mod(step, packing) == 0); + + const ref_i32 = k.memRefBitcastShape(flat, &.{ cfg.kvFlatI32Rows(), cfg.head_dim }, .i32); + const b_stride: i32 = @intCast(@divExact(cfg.numCombinedKvHeadsPerBlk(), packing)); + const b_start: i64 = @divExact(start, packing); + const b = loadStridedOrStack(k, ref_i32, b_start, b_stride, cfg.flashKRows(), cfg.head_dim); + + const splat_shape = &[_]i64{ cfg.flashKRows(), cfg.head_dim }; + + if (cfg.kv_dtype == .bf16) { + const bk = k.shli(b, k.splat(@as(i32, 16), splat_shape, .i32)); + const bv = k.andi(b, k.splat(@as(i32, @bitCast(@as(u32, 0xFFFF0000))), splat_shape, .i32)); + const k_v = k.arithTruncf(bk.bitCast(.f32), cfg.kv_dtype); + const v_v = k.arithTruncf(bv.bitCast(.f32), cfg.kv_dtype); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + const bitwidth: i64 = @divExact(@as(i64, 32), packing); + const trunc_dtype: mtt.DType = switch (bitwidth) { + 16 => .i16, + 8 => .i8, + else => std.debug.panic("ragged_paged: unsupported packing bitwidth {d}", .{bitwidth}), + }; + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + const result = k.arena.allocator().alloc(KvPair, kv_load_step) catch @panic("stridedLoadKv OOM"); + var i: usize = 0; + while (i < kv_load_step) : (i += 1) { + const shift_k: i32 = @intCast(@as(i64, @intCast(i)) * 2 * bitwidth); + const shift_v: i32 = @intCast((@as(i64, @intCast(i)) * 2 + 1) * bitwidth); + const bk = k.shrui(b, k.splat(shift_k, splat_shape, .i32)); + const bv = k.shrui(b, k.splat(shift_v, splat_shape, .i32)); + const bk_trunc = k.trunci(bk, trunc_dtype); + const bv_trunc = k.trunci(bv, trunc_dtype); + result[i] = .{ + .k = bk_trunc.bitCast(cfg.kv_dtype), + .v = bv_trunc.bitCast(cfg.kv_dtype), + }; + } + return result; +} + +/// `flash_attention(q, k, v, ...)` — one online-softmax step over `num_kv_per_blk` +/// keys. +fn flashAttention( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + q: mtt.Value, + kv_k: mtt.Value, + v: mtt.Value, + /// Element type of `kv_k` as currently held — equal to `cfg.kv_dtype` + /// unless `cfg.k_scale` was applied upstream (then `cfg.q_dtype`). + /// Used for the masked-zero round-trip Pallas's `.astype(k.dtype)` does. + k_input_dtype: mtt.DType, + /// Same role as `k_input_dtype`, for `v`. + v_input_dtype: mtt.DType, + kv_head_idx: i64, + q_head_idx: i64, + kv_blk_idx: mtt.Value, + kv_len: mtt.Value, + q_start: mtt.Value, + q_end: mtt.Value, + q_len: mtt.Value, + q_len_start: mtt.Value, +) void { + const flash_q: i64 = cfg.flashQRows(); + const flash_k: i64 = cfg.flashKRows(); + const head_dim: i64 = cfg.head_dim; + const num_q_per_blk: i64 = cfg.numQPerBlk(); + const num_q_heads_per_kv_head: i64 = cfg.numQHeadsPerKvHead(); + + const kv_len_start = k.muli(kv_blk_idx, k.constant(.i32, cfg.numKvPerBlk())); + + const kv_mask = k.cmpi( + .slt, + k.iota(&.{ flash_k, head_dim }, .i32, &.{0}), + k.broadcastTo(k.subi(kv_len, kv_len_start), &.{ flash_k, head_dim }), + ); + const k_zeroed = k.select(kv_mask, kv_k.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(k_input_dtype); + const v_zeroed = k.select(kv_mask, v.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(v_input_dtype); + + k.traceStart(10, "nd,md->nm"); + const qk_raw = k.matmulOpts( + q, + k_zeroed, + k.zeros(&.{ flash_q, flash_k }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{1}, &.{0}, &.{0}, &.{ 0, 0, 1, 0 }, &.{}, &.{}) }, + ); + k.traceStop(); + var qk = k.mulf(qk_raw, k.splat(cfg.sm_scale, &.{ flash_q, flash_k }, .f32)); + + const store_start = k.maxsi(k.subi(q_start, q_len_start), k.constant(.i32, 0)); + const store_end = k.minsi(k.subi(q_end, q_len_start), k.constant(.i32, num_q_per_blk)); + + const row_offset = k.subi(k.addi(k.subi(kv_len, q_len), q_len_start), q_start); + const row_iota = k.iota(&.{ flash_q, flash_k }, .i32, &.{0}); + const row_q = k.divFloor(row_iota, @as(i32, @intCast(num_q_heads_per_kv_head))); + const row_ids = k.addi(k.broadcastTo(row_offset, &.{ flash_q, flash_k }), row_q); + const col_ids = k.addi( + k.broadcastTo(kv_len_start, &.{ flash_q, flash_k }), + k.iota(&.{ flash_q, flash_k }, .i32, &.{1}), + ); + var score_mask = if (cfg.is_causal) + k.cmpi(.slt, row_ids, col_ids) + else + // `row_ids < row_ids` is an all-false mask with the same bool shape + // as the causal mask; sliding-window masking can still be OR-ed below. + k.cmpi(.slt, row_ids, row_ids); + if (cfg.sliding_window) |sw| { + const sw_splat = k.splat(@as(i32, @intCast(sw)), &.{ flash_q, flash_k }, .i32); + const window_mask = k.cmpi(.sge, k.subi(row_ids, sw_splat), col_ids); + score_mask = k.ori(score_mask, window_mask); + } + if (cfg.soft_cap) |sc| { + const sc_splat = k.splat(sc, &.{ flash_q, flash_k }, .f32); + qk = k.mulf(sc_splat, k.tanh(k.divf(qk, sc_splat))); + } + const qk_masked = k.addf( + qk, + k.select(score_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), + ); + + // jnp.max → maximumf (NaN-propagating). + const m_curr_v = k.multiReduction(.maximumf, qk_masked, k.splat(@as(f32, -std.math.inf(f32)), &.{flash_q}, .f32), &.{1}); + const m_curr_2d = k.shapeCast(m_curr_v, &.{ flash_q, 1 }); + const s_curr = k.exp(k.subf(qk_masked, k.broadcastTo(m_curr_2d, &.{ flash_q, flash_k }))); + + const qkv = k.matmulOpts( + s_curr, + v_zeroed, + k.zeros(&.{ flash_q, head_dim }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}) }, + ); + + // Pallas hardcodes the m/l-ref trailing dim to 128 (the TPU lane width) + // regardless of `head_dim`. Broadcast m/l values to `head_dim` only + // immediately before mixing with `qkv` / `o_curr`. + const lm_lane: i64 = 128; + + const m_curr_bc = k.broadcastTo(m_curr_2d, &.{ flash_q, lm_lane }); + const l_curr_v = k.multiReduction(.add, s_curr, k.zeros(&.{flash_q}, .f32), &.{1}); + const l_curr_bc = k.broadcastTo(k.shapeCast(l_curr_v, &.{ flash_q, 1 }), &.{ flash_q, lm_lane }); + + const head_l_ref_idx = k.constIndex(kv_head_idx); + const m_prev_load = k.shapeCast( + k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); + const l_prev_load = k.shapeCast( + k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); + + const m_next = k.maximumf(m_prev, m_curr_bc); + maskedStoreLM(k, a.m_ref, m_next, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const alpha = k.exp(k.subf(m_prev, m_next)); + const beta = k.exp(k.subf(m_curr_bc, m_next)); + const l_alpha = k.mulf(alpha, l_prev); + const l_next = k.addf(l_alpha, k.mulf(beta, l_curr_bc)); + const l_next_safe = k.select( + k.cmpf(.oeq, l_next, k.zeros(&.{ flash_q, lm_lane }, .f32)), + k.splat(@as(f32, 1.0), &.{ flash_q, lm_lane }, .f32), + l_next, + ); + maskedStoreLM(k, a.l_ref, l_next_safe, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const q_head_idx_idx = k.constIndex(q_head_idx); + const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.constIndex(0), q_head_idx_idx, k.constIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + const o_curr_pre = k.select( + k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), + k.zeros(&.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }, .f32), + o_curr_3d, + ); + const o_curr = k.shapeCast(o_curr_pre, &.{ flash_q, head_dim }); + const l_alpha_bc = broadcastToShape(k, l_alpha, &.{ flash_q, head_dim }); + const beta_bc = broadcastToShape(k, beta, &.{ flash_q, head_dim }); + const l_next_safe_bc = broadcastToShape(k, l_next_safe, &.{ flash_q, head_dim }); + const out = k.divf(k.addf(k.mulf(l_alpha_bc, o_curr), k.mulf(beta_bc, qkv)), l_next_safe_bc); + const out_3d = k.shapeCast(out, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + maskedStoreAcc(k, a.acc_ref, out_3d, q_head_idx_idx, store_start, store_end); +} + +/// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the +/// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ + .{ .program_id = 1 }, + .{ .program_id = 0 }, + .zero, + }; + + const a = try b.declareArgsOpts(.{ + .heads_blk_idx = .{ .scalar = .i32 }, + .q_blk_idx = .{ .scalar = .i32 }, + + .kv_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .page_indices = .{ .ref = .{ .shape = &.{ cfg.max_num_seqs, cfg.pages_per_seq }, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .cu_q_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs + 1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .seq_buf_idx = .{ .ref = .{ .shape = &.{2}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .num_seqs = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + + .q = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + .kv_pages = .{ .ref = .{ + .shape = &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeads(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .memory_space = .any, + } }, + .o = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .role = .output, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + + .kv_bufs = .{ .ref = .{ + .shape = &.{ 2, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .role = .scratch, + } }, + .sems = .{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }, + // Pallas hardcodes the trailing dim to 128 (TPU lane width); see + // `kernel.py:lm_scratch`. + .l_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .m_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .acc_ref = .{ .ref = .{ .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .dtype = .f32, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{ .arbitrary, .arbitrary }, + .iteration_bounds = &.{ cfg.numHeadsBlks(), cfg.numQBlks() }, + .scalar_prefetch = 5, + .scratch_operands = 5, + .pallas_window_params = true, + }); + + const k = b; + + const num_seqs = k.scalarLoad(a.num_seqs, &.{k.constIndex(0)}); + const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(0)}); + const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(1)}); + const q_len_start = k.muli(a.q_blk_idx, k.constant(.i32, cfg.numQPerBlk())); + const q_len_end = k.addi(q_len_start, k.constant(.i32, cfg.numQPerBlk())); + + var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.constant(.i32, 0))); + { + const first_seq_idx = nextNonemptyKvSequence(k, cfg, a, init_seq_idx, num_seqs); + var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); + { + createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.constant(.i32, 0), init_buf_idx); + nonempty.close(); + } + i.close(); + } + + // Empty rows skip flashAttention, so their accumulator must already be zero. + k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); + + var q_loop = k.openWhile(.{ k.constant(.i32, 0), init_seq_idx, init_buf_idx }, .{ .i32, .i32, .i32 }); + { + const done = q_loop.before_carried[0]; + const cur_seq_idx = q_loop.before_carried[1]; + const cur_buf_idx = q_loop.before_carried[2]; + const cu_at_num_seqs = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(num_seqs)}); + const should_run = k.andi( + k.cmpi(.slt, q_len_start, cu_at_num_seqs), + k.cmpi(.slt, cur_seq_idx, num_seqs), + ); + const cond = k.andi(k.cmpi(.eq, done, k.constant(.i32, 0)), should_run); + q_loop.yieldBefore(cond, .{ done, cur_seq_idx, cur_buf_idx }); + } + { + const done = q_loop.after_carried[0]; + const cur_seq_idx = q_loop.after_carried[1]; + const cur_buf_idx = q_loop.after_carried[2]; + + const q_start = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(cur_seq_idx)}); + const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.constant(.i32, 1)))}); + const q_len = k.subi(q_end, q_start); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(cur_seq_idx)}); + + var kv_loop = k.openWhile(.{ k.constant(.i32, 0), cur_buf_idx }, .{ .i32, .i32 }); + { + const kv_blk = kv_loop.before_carried[0]; + const buf = kv_loop.before_carried[1]; + const cond = k.cmpi(.slt, k.muli(kv_blk, k.constant(.i32, cfg.numKvPerBlk())), kv_len); + kv_loop.yieldBefore(cond, .{ kv_blk, buf }); + } + { + const kv_blk = kv_loop.after_carried[0]; + const cur_buf = kv_loop.after_carried[1]; + + const next_kv_blk_raw = k.addi(kv_blk, k.constant(.i32, 1)); + const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.constant(.i32, cfg.numKvPerBlk())), kv_len); + const next_kv_blk = k.select(is_last_kv, k.constant(.i32, 0), next_kv_blk_raw); + const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); + const next_nonempty_seq = blk: { + // Only sequence transitions need to scan for empty KV rows. + var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{.i32}); + { + const start_seq_idx = k.addi(cur_seq_idx, k.constant(.i32, 1)); + next_seq.yieldThen(.{nextNonemptyKvSequence(k, cfg, a, start_seq_idx, num_seqs)}); + } + { + next_seq.yieldElse(.{cur_seq_idx}); + } + break :blk next_seq.results[0]; + }; + const is_last_seq = k.cmpi(.eq, next_nonempty_seq, num_seqs); + const next_seq_idx = k.select(is_last_seq, k.constant(.i32, 0), next_nonempty_seq); + const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.constant(.i32, 1)), a.heads_blk_idx); + const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.constant(.i32, 0)), .i32); + + var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.constant(.i32, cfg.numHeadsBlks()))); + { + createKvAsyncCopyDescriptors(k, cfg, a, next_heads_blk_idx, next_seq_idx, next_kv_blk, next_buf); + ii.close(); + } + + waitKvDma(k, cfg, a, a.heads_blk_idx, cur_seq_idx, kv_blk, cur_buf); + + // for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step) — unrolled at IR-emit time. + const num_kv_heads_per_blk: usize = @intCast(cfg.numKvHeadsPerBlk()); + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + var kv_head_chunk_idx: usize = 0; + while (kv_head_chunk_idx < num_kv_heads_per_blk) : (kv_head_chunk_idx += kv_load_step) { + const kv_pairs = stridedLoadKv(k, cfg, a, cur_buf, @intCast(kv_head_chunk_idx)); + var step_idx: usize = 0; + while (step_idx < kv_load_step) : (step_idx += 1) { + var kv_k_val = kv_pairs[step_idx].k; + var kv_v_val = kv_pairs[step_idx].v; + var k_input_dtype = cfg.kv_dtype; + var v_input_dtype = cfg.kv_dtype; + if (cfg.k_scale) |ks| { + const ks_splat = k.splat(ks, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_k_val = k.mulf(kv_k_val.to(.f32), ks_splat).to(cfg.q_dtype); + k_input_dtype = cfg.q_dtype; + } + if (cfg.v_scale) |vs| { + const vs_splat = k.splat(vs, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_v_val = k.mulf(kv_v_val.to(.f32), vs_splat).to(cfg.q_dtype); + v_input_dtype = cfg.q_dtype; + } + const kv_head_idx: i64 = @intCast(kv_head_chunk_idx + step_idx); + const q_head_idx = kv_head_idx * cfg.numQHeadsPerKvHead(); + + // `fold_on_2nd_minor` (kernel.py:255-262): when the + // 2nd-to-last dim is not a multiple of q's packing + // factor, cast q to f32 before the trailing-dim reshape + // so the layout pass can fold it cleanly. + var q_block = k.vectorLoadShape(a.q, &.{ k.constIndex(0), k.constIndex(q_head_idx), k.constIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); + if (@mod(cfg.numQHeadsPerKvHead(), cfg.qPackingFactor()) != 0) { + q_block = q_block.to(.f32); + } + const q_2d = k.shapeCast(q_block, &.{ cfg.flashQRows(), cfg.head_dim }); + flashAttention(k, cfg, a, q_2d, kv_k_val, kv_v_val, k_input_dtype, v_input_dtype, kv_head_idx, q_head_idx, kv_blk, kv_len, q_start, q_end, q_len, q_len_start); + } + } + + kv_loop.yieldAfter(.{ k.addi(kv_blk, k.constant(.i32, 1)), next_buf }); + } + const next_buf_idx = kv_loop.results[1]; + + const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.constant(.i32, 1)), cur_seq_idx); + const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.constant(.i32, 1)); + q_loop.yieldAfter(.{ next_done, next_seq_idx_outer, next_buf_idx }); + } + const seq_idx_out = q_loop.results[1]; + const buf_idx_out = q_loop.results[2]; + + k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.constant(.i32, 0)), &.{k.constIndex(0)}); + k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.constIndex(1)}); + + k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); +} + +// Fixed drafter blocks can have query tokens without visible KV tokens. +// Keep their query positions, but never schedule a DMA for an empty KV row. +fn nextNonemptyKvSequence( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + start_seq_idx: mtt.Value, + num_seqs: mtt.Value, +) mtt.Value { + var loop = k.openWhile(.{start_seq_idx}, .{.i32}); + { + const seq_idx = loop.before_carried[0]; + // The terminal index can equal the metadata array length. + const safe_seq_idx = k.minsi(seq_idx, k.constant(.i32, cfg.max_num_seqs - 1)); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(safe_seq_idx)}); + const skip = k.andi( + k.cmpi(.slt, seq_idx, num_seqs), + k.cmpi(.sle, kv_len, k.constant(.i32, 0)), + ); + loop.yieldBefore(skip, .{seq_idx}); + } + { + loop.yieldAfter(.{k.addi(loop.after_carried[0], k.constant(.i32, 1))}); + } + return loop.results[0]; +} + +/// Open a builder, run the kernel body, and return the canonicalized MLIR IR. +/// Used by offline tooling (`examples/mosaic_ragged_paged`); production model +/// code goes through `zml.kernel.mosaic_tpu.Kernel(...).emit(...)` instead. +pub fn buildIr(allocator: std.mem.Allocator, ctx: *mlir.Context, cfg: Cfg) ![:0]const u8 { + var b = try mtt.Builder.open(allocator, ctx, "ragged_paged_attention_kernel"); + defer b.deinit(); + try run(&b, cfg); + return b.finishOpts(&.{}, .{ .canonicalize = true }); +} diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index fe111015..2fbc2a9b 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -1,17 +1,17 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( - name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.33.dev20251218+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", + name = "libzml_tpu", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.50.dev20261005+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "b7d9855c74a7fc56d11c4c7a2c55d5395c80594d43caec1549c69d5f908583cf", - build_file = "libpjrt_tpu.BUILD.bazel", + sha256 = "a1e8615a33f60e1bfce15ec9d570642f882a8bec1d46763531e3b831edb2bb42", + build_file = "libzml_tpu.BUILD.bazel", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_tpu"], + root_module_direct_deps = ["libzml_tpu"], root_module_direct_dev_deps = [], ) diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index a9316699..3c8143bb 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -3,15 +3,14 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); const pjrt = @import("pjrt"); -const runfiles = @import("runfiles"); +const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/tpu"); pub fn isEnabled() bool { - return @hasDecl(c, "ZML_RUNTIME_TPU"); + return platforms_options.tpu_enabled; } /// Check if running on Google Compute Engine, because TPUs will poll the @@ -34,7 +33,7 @@ fn isOnGCP(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; // autofix + _ = allocator; if (comptime !isEnabled()) { return error.Unavailable; } @@ -48,14 +47,14 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libzml_tpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for TPU runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_tpu.so" }); break :blk .loadFrom(path); }; } diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index f42a73fa..d7c94d82 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -9,17 +9,18 @@ zig_library( "debug.zig", "flags.zig", "fmt.zig", + "cpu.zig", "json.zig", "math.zig", "meta.zig", "process.zig", "queue.zig", "segmented_list.zig", - "signature.zig", "time.zig", "unicode.zig", ], main = "stdx.zig", + tags = ["manual"], visibility = ["//visibility:public"], zigopts = ["-lc"], ) @@ -29,3 +30,9 @@ zig_test( testonly = False, deps = [":stdx"], ) + +zig_test( + name = "cpu_test", + main = "cpu.zig", + zigopts = ["-lc"], +) diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index bdee5bee..1f77a14d 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -27,12 +27,16 @@ pub fn BoundedArray(comptime T: type, comptime buffer_capacity: usize) type { /// only known at runtime, but whose maximum size is known at comptime, without /// requiring an `Allocator`. /// ```zig -// var a = try BoundedArrayAligned(u8, 16, 2).init(0); -// try a.append(255); -// try a.append(255); -// const b = @ptrCast(*const [1]u16, a.constSlice().ptr); -// try testing.expectEqual(@as(u16, 65535), b[0]); +/// var a: BoundedArrayAligned(u8, 16, 2) = .empty; +/// try a.append(255); +/// try a.append(255); +/// const b: *const [1]u16 = @ptrCast(a.constSlice().ptr); +/// try testing.expectEqual(@as(u16, 65535), b[0]); /// ``` +/// +/// Note: there is no init anymore. It's generally unsafe because it makes `slice()` return undefined.; +/// If you used `try init(0)` use `.empty` +/// If you used `try init(n)` use `.{ .buffer = undefined, .len = n }` and fill `buffer` yourself. pub fn BoundedArrayAligned( comptime T: type, comptime alignment: Alignment, @@ -40,14 +44,24 @@ pub fn BoundedArrayAligned( ) type { return struct { const Self = @This(); - buffer: [buffer_capacity]T align(alignment.toByteUnits()) = undefined, - len: usize = 0, + buffer: [buffer_capacity]T align(alignment.toByteUnits()), + len: usize, - /// Set the actual length of the slice. - /// Returns error.Overflow if it exceeds the length of the backing array. - pub fn init(len: usize) error{Overflow}!Self { - if (len > buffer_capacity) return error.Overflow; - return Self{ .len = len }; + pub const max_len: usize = buffer_capacity; + + pub const empty: Self = .{ .buffer = undefined, .len = 0 }; + + pub fn init(m: []const T) Self { + var buf: Self = .{ .buffer = undefined, .len = m.len }; + @memcpy(buf.buffer[0..m.len], m); + return buf; + } + + pub fn repeat(value: T, len: usize) Self { + std.debug.assert(len <= buffer_capacity); + var buf: Self = .{ .buffer = undefined, .len = len }; + @memset(buf.buffer[0..len], value); + return buf; } /// View the internal array as a slice whose size was previously set. @@ -78,19 +92,20 @@ pub fn BoundedArrayAligned( /// Copy the content of an existing slice. pub fn fromSlice(m: []const T) error{Overflow}!Self { - var list = try init(m.len); - @memcpy(list.slice(), m); - return list; + if (m.len > buffer_capacity) return error.Overflow; + return .init(m); } /// Return the element at index `i` of the slice. pub fn get(self: Self, i: usize) T { - return self.constSlice()[i]; + std.debug.assert(i < self.len); + return self.buffer[i]; } /// Set the value of the element at index `i` of the slice. pub fn set(self: *Self, i: usize, item: T) void { - self.slice()[i] = item; + std.debug.assert(i < self.len); + self.buffer[i] = item; } /// Return the maximum length of a slice. @@ -273,17 +288,6 @@ pub fn BoundedArrayAligned( @memset(self.slice()[old_len..self.len], value); } - //pub const Writer = if (T != u8) - // @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ - // "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") - //else - // std.io.GenericWriter(*Self, error{Overflow}, appendWrite); - - ///// Initializes a writer which will write into the array. - //pub fn writer(self: *Self) Writer { - // return .{ .context = self }; - //} - /// Same as `appendSlice` except it returns the number of bytes written, which is always the same /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize { @@ -294,7 +298,7 @@ pub fn BoundedArrayAligned( } test BoundedArray { - var a = try BoundedArray(u8, 64).init(32); + var a: BoundedArray(u8, 64) = .{ .buffer = undefined, .len = 32 }; try testing.expectEqual(a.capacity(), 64); try testing.expectEqual(a.slice().len, 32); @@ -303,8 +307,8 @@ test BoundedArray { try a.resize(48); try testing.expectEqual(a.len, 48); - const x = [_]u8{1} ** 10; - a = try BoundedArray(u8, 64).fromSlice(&x); + const x: [10]u8 = @splat(1); + a = try .fromSlice(&x); try testing.expectEqualSlices(u8, &x, a.constSlice()); var a2 = a; @@ -393,14 +397,13 @@ test BoundedArray { try testing.expectEqual(a.len, 36); while (a.pop()) |_| {} - const w = a.writer(); const s = "hello, this is a test string"; - try w.writeAll(s); + _ = try a.appendWrite(s); try testing.expectEqualStrings(s, a.constSlice()); } test "BoundedArrayAligned" { - var a = try BoundedArrayAligned(u8, .@"16", 4).init(0); + var a: BoundedArrayAligned(u8, .@"16", 4) = .empty; try a.append(0); try a.append(0); try a.append(255); diff --git a/stdx/cpu.zig b/stdx/cpu.zig new file mode 100644 index 00000000..c51b32d4 --- /dev/null +++ b/stdx/cpu.zig @@ -0,0 +1,411 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +/// Restricts the calling thread and existing threads to the CPUs of a NUMA +/// node within the caller's current affinity, preserving each thread's narrower +/// affinity. Threads with no overlap are left alone. Returns the number of +/// threads whose affinity changed. +/// Call during startup, before threads can concurrently create other threads. +/// On failure, previously changed affinities remain in effect. +pub fn pinProcessToNumaNode(io: std.Io, node: u32) !usize { + if (builtin.os.tag != .linux) return error.Unsupported; + + var path_buf: [64]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/node/node{d}/cpulist", .{node}); + + var cpulist_buf: [4096]u8 = undefined; + const cpulist = try std.Io.Dir.cwd().readFile(io, path, &cpulist_buf); + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + + return pinProcessToCpuSet(io, set); +} + +/// CPUs the calling thread may run on, ordered for worker pools. +pub const AllowedCores = struct { + /// Allowed CPU ids: one per physical core first, then the remaining SMT + /// siblings, round by round. + ids: []const usize, + /// `ids[0..physical_count]` holds exactly one CPU per physical core. Equals + /// `ids.len` when the topology is unknown. + physical_count: usize, + + /// One CPU per physical core, skipping SMT siblings. + pub fn physical(self: AllowedCores) []const usize { + return self.ids[0..self.physical_count]; + } + + pub fn deinit(self: AllowedCores, allocator: std.mem.Allocator) void { + allocator.free(self.ids); + } +}; + +/// Returns the CPUs the calling thread may run on. Linux uses sysfs topology so +/// that a second logical CPU of a physical core comes only after every allowed +/// physical core got one. +pub fn allowedCores(allocator: std.mem.Allocator, io: std.Io) !AllowedCores { + if (builtin.os.tag != .linux) return naturalOrder(allocator, try std.Thread.getCpuCount()); + + const cpus = linuxAllowedCoreIds(allocator) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return naturalOrder(allocator, try std.Thread.getCpuCount()), + }; + errdefer allocator.free(cpus); + + const physical_count = linuxOrderBySiblings(allocator, io, cpus) catch |err| switch (err) { + error.OutOfMemory => return err, + else => cpus.len, + }; + return .{ .ids = cpus, .physical_count = physical_count }; +} + +fn pinProcessToCpuSet(io: std.Io, node_set: std.os.linux.cpu_set_t) !usize { + const linux = std.os.linux; + // Keep the caller's taskset/numactl choice as the process-wide limit, and + // intersect each thread's own affinity below so private bindings stay narrow. + const set = intersectCpuSets(node_set, try threadAffinity(0)); + if (std.mem.allEqual(usize, &set, 0)) return 0; + + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + + // Pin the caller first so threads it creates inherit the target mask. + var pinned: usize = @intFromBool(try pinThreadToCpuSet(0, set)); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(linux.pid_t, entry.name, 10) catch continue; + // Threads may exit while the list is being walked. + const changed = pinThreadToCpuSet(tid, set) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + pinned += @intFromBool(changed); + } + + return pinned; +} + +fn intersectCpuSets(a: std.os.linux.cpu_set_t, b: std.os.linux.cpu_set_t) std.os.linux.cpu_set_t { + var set = a; + for (&set, b) |*word, other| word.* &= other; + return set; +} + +fn threadAffinity(tid: std.os.linux.pid_t) error{ ThreadGone, GetAffinityFailed }!std.os.linux.cpu_set_t { + const linux = std.os.linux; + var set: linux.cpu_set_t = @splat(0); + switch (linux.errno(linux.sched_getaffinity(tid, @sizeOf(linux.cpu_set_t), &set))) { + .SUCCESS => return set, + .SRCH => return error.ThreadGone, + else => return error.GetAffinityFailed, + } +} + +fn setThreadAffinity(tid: std.os.linux.pid_t, set: *const std.os.linux.cpu_set_t) error{ ThreadGone, SetAffinityFailed }!void { + const linux = std.os.linux; + const rc = linux.syscall3(.sched_setaffinity, @as(usize, @bitCast(@as(isize, tid))), @sizeOf(linux.cpu_set_t), @intFromPtr(set)); + switch (linux.errno(rc)) { + .SUCCESS => {}, + .SRCH => return error.ThreadGone, + else => return error.SetAffinityFailed, + } +} + +fn pinThreadToCpuSet(tid: std.os.linux.pid_t, target: std.os.linux.cpu_set_t) !bool { + const current = try threadAffinity(tid); + const set = intersectCpuSets(target, current); + if (std.mem.allEqual(usize, &set, 0) or std.mem.eql(usize, &set, ¤t)) return false; + try setThreadAffinity(tid, &set); + return true; +} + +/// Parses Linux CPU lists, including singleton CPUs and inclusive ranges. +fn parseCpuList(cpulist: []const u8, set: *std.os.linux.cpu_set_t) !void { + const trimmed = std.mem.trim(u8, cpulist, " \t\r\n"); + if (trimmed.len == 0) return; + + const bits_per_word = @bitSizeOf(usize); + + var ranges = std.mem.splitScalar(u8, trimmed, ','); + while (ranges.next()) |raw_range| { + const range = std.mem.trim(u8, raw_range, " \t\r\n"); + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return error.InvalidCpuList; + + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return error.InvalidCpuList + else + first; + + if (last < first or ends.next() != null or last >= set.len * bits_per_word) return error.InvalidCpuList; + for (first..last + 1) |cpu| { + set[cpu / bits_per_word] |= @as(usize, 1) << @intCast(cpu % bits_per_word); + } + } +} + +fn linuxAllowedCoreIds(allocator: std.mem.Allocator) ![]usize { + const set = try threadAffinity(0); + + var core_ids: std.ArrayList(usize) = .empty; + errdefer core_ids.deinit(allocator); + + var core_id: usize = 0; + while (core_id < @bitSizeOf(std.os.linux.cpu_set_t)) : (core_id += 1) { + if (cpuSetContains(set, core_id)) { + try core_ids.append(allocator, core_id); + } + } + + if (core_ids.items.len == 0) return error.MissingCpuTopology; + + return core_ids.toOwnedSlice(allocator); +} + +fn cpuSetContains(set: std.os.linux.cpu_set_t, core_id: usize) bool { + const word_bit_count = @bitSizeOf(usize); + const word_index = core_id / word_bit_count; + const bit_index: std.math.Log2Int(usize) = @intCast(core_id % word_bit_count); + return word_index < set.len and (set[word_index] & (@as(usize, 1) << bit_index)) != 0; +} + +fn linuxOrderBySiblings(allocator: std.mem.Allocator, io: std.Io, cpus: []usize) !usize { + const leaders = try allocator.alloc(usize, cpus.len); + defer allocator.free(leaders); + for (cpus, leaders) |cpu, *leader| leader.* = try linuxCoreLeader(io, cpu); + return orderBySiblings(allocator, cpus, leaders); +} + +/// Returns the lowest CPU id sharing a physical core with `cpu`. +fn linuxCoreLeader(io: std.Io, cpu: usize) !usize { + var cpulist_buf: [4096]u8 = undefined; + // `core_cpus_list` replaced `thread_siblings_list` in Linux 5.7. + const cpulist = linuxReadTopology(io, cpu, "core_cpus_list", &cpulist_buf) catch |err| switch (err) { + error.FileNotFound => try linuxReadTopology(io, cpu, "thread_siblings_list", &cpulist_buf), + else => return err, + }; + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + for (set, 0..) |word, word_index| { + if (word != 0) return word_index * @bitSizeOf(usize) + @ctz(word); + } + return error.MissingCpuTopology; +} + +fn linuxReadTopology(io: std.Io, cpu: usize, name: []const u8, buf: []u8) ![]const u8 { + var path_buf: [96]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/cpu/cpu{d}/topology/{s}", .{ cpu, name }); + return std.Io.Dir.cwd().readFile(io, path, buf); +} + +/// Reorders `cpus` so that each round holds at most one CPU per physical core, +/// in ascending order, before the next round of SMT siblings. `leaders[i]` +/// identifies the physical core of `cpus[i]`. Returns the number of physical +/// cores, which is also the size of the first round. +fn orderBySiblings(allocator: std.mem.Allocator, cpus: []usize, leaders: []const usize) !usize { + const Entry = struct { + round: usize, + cpu: usize, + + fn lessThan(_: void, lhs: @This(), rhs: @This()) bool { + return if (lhs.round != rhs.round) lhs.round < rhs.round else lhs.cpu < rhs.cpu; + } + }; + + const entries = try allocator.alloc(Entry, cpus.len); + defer allocator.free(entries); + + var physical_count: usize = 0; + for (entries, cpus, leaders, 0..) |*entry, cpu, leader, index| { + var round: usize = 0; + for (leaders[0..index]) |previous| round += @intFromBool(previous == leader); + entry.* = .{ .round = round, .cpu = cpu }; + physical_count += @intFromBool(round == 0); + } + + std.mem.sort(Entry, entries, {}, Entry.lessThan); + for (cpus, entries) |*cpu, entry| cpu.* = entry.cpu; + return physical_count; +} + +fn naturalOrder(allocator: std.mem.Allocator, cpu_count: usize) !AllowedCores { + const ids = try allocator.alloc(usize, cpu_count); + for (ids, 0..) |*id, index| id.* = index; + return .{ .ids = ids, .physical_count = ids.len }; +} + +test "CPU lists respect mask boundaries" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList("0-2,64,66-67\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); + const capacity = @bitSizeOf(std.os.linux.cpu_set_t); + var text: [32]u8 = undefined; + try parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity - 1}), &set); + try std.testing.expect((set[set.len - 1] & (@as(usize, 1) << (@bitSizeOf(usize) - 1))) != 0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity}), &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("3-1", &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("1-2-3", &set)); +} + +const AffinityTest = struct { + const CpuSet = std.os.linux.cpu_set_t; + + fn available(count: usize) !CpuSet { + if (builtin.os.tag != .linux) return error.SkipZigTest; + const current = try threadAffinity(0); + var selected: CpuSet = @splat(0); + var remaining = count; + for (0..@bitSizeOf(CpuSet)) |core_id| { + if (!cpuSetContains(current, core_id)) continue; + selected[core_id / @bitSizeOf(usize)] |= @as(usize, 1) << @intCast(core_id % @bitSizeOf(usize)); + remaining -= 1; + if (remaining == 0) break; + } + if (remaining != 0) return error.SkipZigTest; + return selected; + } + + fn check(caller: CpuSet, worker: CpuSet, node: CpuSet, expected: CpuSet) !void { + const io = std.testing.io; + const Worker = struct { + io: std.Io, + set: CpuSet, + tid: std.os.linux.pid_t = undefined, + ready: std.Io.Event = .unset, + stop: std.Io.Event = .unset, + failure: ?anyerror = null, + + fn run(self: *@This()) void { + self.tid = @intCast(std.Thread.getCurrentId()); + setThreadAffinity(0, &self.set) catch |err| { + self.failure = err; + }; + self.ready.set(self.io); + self.stop.waitUncancelable(self.io); + } + }; + var state: Worker = .{ .io = io, .set = worker }; + const thread = try std.Thread.spawn(.{}, Worker.run, .{&state}); + defer { + state.stop.set(io); + thread.join(); + } + try state.ready.wait(io); + if (state.failure) |err| return err; + + // Restore every thread touched by the process-wide helper, including + // any test runner IO workers, before the next test executes. + var saved: std.ArrayList(struct { tid: std.os.linux.pid_t, set: CpuSet }) = .empty; + defer { + for (saved.items) |entry| setThreadAffinity(entry.tid, &entry.set) catch |err| switch (err) { + error.ThreadGone => {}, + error.SetAffinityFailed => @panic("failed to restore test thread affinity"), + }; + saved.deinit(std.testing.allocator); + } + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(std.os.linux.pid_t, entry.name, 10) catch continue; + const set = threadAffinity(tid) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + try saved.append(std.testing.allocator, .{ .tid = tid, .set = set }); + } + + try setThreadAffinity(0, &caller); + _ = try pinProcessToCpuSet(io, node); + try std.testing.expectEqualSlices(usize, &expected, &(try threadAffinity(state.tid))); + const expected_caller = intersectCpuSets(caller, node); + try std.testing.expectEqualSlices(usize, &expected_caller, &(try threadAffinity(0))); + } +}; + +test "process NUMA affinity preserves a narrower worker binding" { + const caller = try AffinityTest.available(3); + const node = try AffinityTest.available(2); + const worker = try AffinityTest.available(1); + try AffinityTest.check(caller, worker, node, worker); +} + +test "process NUMA affinity visits workers when the caller is already pinned" { + const caller = try AffinityTest.available(1); + const worker = try AffinityTest.available(2); + try AffinityTest.check(caller, worker, worker, caller); +} + +test "process NUMA affinity leaves a worker with no overlapping CPUs alone" { + const caller = try AffinityTest.available(2); + const node = try AffinityTest.available(1); + var worker = caller; + for (&worker, node) |*word, excluded| word.* &= ~excluded; + try AffinityTest.check(caller, worker, node, worker); +} + +test "CPU lists accept whitespace and empty node CPU sets" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(" \t\r\n", &set); + try std.testing.expect(std.mem.allEqual(usize, &set, 0)); + try parseCpuList(" \t0-2, 64,66-67\r\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); +} + +test "CPU lists reject malformed ranges" { + for ([_][]const u8{ "3-1", "x", "-1", "1-", "1-2-3", "1,,2", "1,", ",1", "999999999999999999999999999999" }) |text| { + var set: std.os.linux.cpu_set_t = @splat(0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(text, &set)); + } +} + +test "natural order treats every CPU as physical" { + const cores = try naturalOrder(std.testing.allocator, 4); + defer cores.deinit(std.testing.allocator); + + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.ids); + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.physical()); +} + +test "sibling order puts one CPU per physical core first" { + const Case = struct { cpus: []const usize, leaders: []const usize, expected: []const usize, physical_count: usize }; + const cases = [_]Case{ + // Siblings numbered after all cores (typical Intel). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 1, 0, 1 }, .expected = &.{ 0, 1, 2, 3 }, .physical_count = 2 }, + // Adjacent siblings (typical AMD, some ARM). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 0, 2, 2 }, .expected = &.{ 0, 2, 1, 3 }, .physical_count = 2 }, + // Affinity allows only some siblings of cores {0,4}, {1,5}, {2,6}. + .{ .cpus = &.{ 0, 1, 2, 5 }, .leaders = &.{ 0, 1, 2, 1 }, .expected = &.{ 0, 1, 2, 5 }, .physical_count = 3 }, + // Affinity allows only the second sibling of some cores. + .{ .cpus = &.{ 1, 4, 5 }, .leaders = &.{ 1, 0, 1 }, .expected = &.{ 1, 4, 5 }, .physical_count = 2 }, + // No SMT. + .{ .cpus = &.{ 0, 1, 2 }, .leaders = &.{ 0, 1, 2 }, .expected = &.{ 0, 1, 2 }, .physical_count = 3 }, + }; + + for (cases) |case| { + const cpus = try std.testing.allocator.dupe(usize, case.cpus); + defer std.testing.allocator.free(cpus); + + const physical_count = try orderBySiblings(std.testing.allocator, cpus, case.leaders); + try std.testing.expectEqualSlices(usize, case.expected, cpus); + try std.testing.expectEqual(case.physical_count, physical_count); + } +} + +test "allowed cores cover the thread affinity" { + const cores = try allowedCores(std.testing.allocator, std.testing.io); + defer cores.deinit(std.testing.allocator); + + try std.testing.expect(cores.physical_count > 0 and cores.physical_count <= cores.ids.len); + if (builtin.os.tag != .linux) return; + + const affinity = try threadAffinity(0); + var expected_count: usize = 0; + for (affinity) |word| expected_count += @popCount(word); + try std.testing.expectEqual(expected_count, cores.ids.len); + for (cores.ids) |cpu| try std.testing.expect(cpuSetContains(affinity, cpu)); +} diff --git a/stdx/debug.zig b/stdx/debug.zig index 4686b3b5..a217eca1 100644 --- a/stdx/debug.zig +++ b/stdx/debug.zig @@ -10,11 +10,12 @@ pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytyp pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { if (!check) { + @branchHint(.cold); panic(msg, args); } } -pub inline fn panic(comptime format: []const u8, args: anytype) noreturn { +pub fn panic(comptime format: []const u8, args: anytype) noreturn { std.debug.panic(format, args); } diff --git a/stdx/flags.zig b/stdx/flags.zig index 9a0f32ae..d0c4352d 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -93,8 +93,9 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) } fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { - comptime assert(@typeInfo(Commands) == .Union); - comptime assert(std.meta.fields(Commands).len >= 2); + comptime assert(@typeInfo(Commands) == .@"union"); + const field_names = std.meta.fieldNames(Commands); + comptime assert(field_names.len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -104,15 +105,15 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com // NB: help must be declared as *pub* const to be visible here. if (@hasDecl(Commands, "help")) { if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { - std.io.getStdOut().writeAll(Commands.help) catch std.process.exit(1); + std.debug.print(Commands.help, .{}); std.process.exit(0); } } - inline for (comptime std.meta.fields(Commands)) |field| { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - if (std.mem.eql(u8, first_arg, field.name)) { - return @unionInit(Commands, field.name, parse_flags(args, field.type)); + inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + if (std.mem.eql(u8, first_arg, field_name)) { + return @unionInit(Commands, field_name, parse_flags(args, FieldType)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); @@ -128,52 +129,61 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { return {}; } - assert(@typeInfo(Flags) == .@"struct"); + const flags_info = @typeInfo(Flags).@"struct"; - comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; + const total_num_fields = flags_info.field_names.len; + comptime var field_names: [total_num_fields][]const u8 = undefined; + comptime var field_types: [total_num_fields]type = undefined; + comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; comptime var field_count = 0; - comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; + const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) + @typeInfo(@FieldType(Flags, "positional")).@"struct" + else + null; - comptime for (std.meta.fields(Flags)) |field| { - if (std.mem.eql(u8, field.name, "positional")) { - assert(@typeInfo(field.type) == .@"struct"); - positional_fields = std.meta.fields(field.type); + comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { + if (std.mem.eql(u8, field_name, "positional")) { var optional_tail = false; - for (positional_fields) |positional_field| { - if (default_value(positional_field) == null) { + for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { + const default: ?PosType = pos_attr.defaultValue(PosType); + if (default == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(positional_field.type)) { + switch (@typeInfo(PosType)) { .optional => |optional| { // optional flags should have a default - assert(default_value(positional_field) != null); - assert(default_value(positional_field).? == null); + assert(default != null); + assert(default.? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(positional_field.type); + assert_valid_value_type(PosType); }, } } } else { - fields[field_count] = field; + field_names[field_count] = field_name; + field_types[field_count] = FieldType; + field_defaults[field_count] = field_attr.default_value_ptr; field_count += 1; - switch (@typeInfo(field.type)) { + const default: ?FieldType = field_attr.defaultValue(FieldType); + + switch (@typeInfo(FieldType)) { .bool => { // boolean flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == false, "boolean flag --{s} should default to false", .{field.name}); + debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); }, .optional => |optional| { // optional flags should have a default - debug.assertComptime(default_value(field) != null and default_value(field).? == null, "optional flag --{s} should have a null default value", .{field.name}); + debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(field.type); + assert_valid_value_type(FieldType); }, } } @@ -182,29 +192,33 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - const f = std.meta.fields(Flags); - var count_field_names: [fields.len][]const u8 = undefined; - var count_field_types: [fields.len]type = undefined; - var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { - field_name.* = field.name; - field_type.* = u32; - field_attr.* = .{ - .@"align" = @alignOf(u32), - .default_value_ptr = @ptrCast(&@as(u32, 0)), - }; - } - break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; + const count_field_types: [total_num_fields]type = @splat(u32); + const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }); + break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for (fields[0..field_count], 0..) |*field_right, i| { - for (fields[0..i]) |*field_left| { - if (field_left.name.len < field_right.name.len) { - std.mem.swap(std.builtin.Type.StructField, field_left, field_right); + for ( + 0.., + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |i, *name_right, *type_right, *default_right| { + for ( + field_names[0..i], + field_types[0..i], + field_defaults[0..i], + ) |*name_left, *type_left, *default_left| { + if (name_left.len < name_right.len) { + std.mem.swap([]const u8, name_left, name_right); + std.mem.swap(type, type_left, type_right); + std.mem.swap(?*const anyopaque, default_left, default_right); } } } @@ -212,30 +226,38 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var parsed_positional = false; next_arg: while (args.next()) |arg| { - comptime var field_len_prev = std.math.maxInt(usize); - inline for (fields[0..field_count]) |field| { - const flag = comptime flag_name(field); + // NB: help must be declared as *pub* const to be visible here. + if (@hasDecl(Flags, "help")) { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + std.debug.print(Flags.help, .{}); + std.process.exit(0); + } + } - comptime assert(field_len_prev >= field.name.len); - field_len_prev = field.name.len; + comptime var field_len_prev = std.math.maxInt(usize); + inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { + const flag = comptime flag_name(field_name); + + comptime assert(field_len_prev >= field_name.len); + field_len_prev = field_name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field.name) += 1; - const flag_value = parse_flag(field.type, flag, arg); - @field(result, field.name) = flag_value; + @field(counts, field_name) += 1; + const flag_value = parse_flag(FieldType, flag, arg); + @field(result, field_name) = flag_value; continue :next_arg; } } - if (@hasField(Flags, "positional")) { + if (positional_infos) |pos_infos| { counts.positional += 1; switch (counts.positional - 1) { - inline 0...positional_fields.len - 1 => |positional_index| { - const positional_field = positional_fields[positional_index]; - const flag = comptime flag_name_positional(positional_field); + inline 0...pos_infos.field_names.len - 1 => |positional_index| { + const positional_field_name = pos_infos.field_names[positional_index]; + const flag = comptime flag_name_positional(positional_field_name); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -244,8 +266,8 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field.name) = - parse_value(positional_field.type, flag, arg); + @field(result.positional, positional_field_name) = + parse_value(pos_infos.field_types[positional_index], flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -255,11 +277,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for (fields[0..field_count]) |field| { - const flag = flag_name(field); - switch (@field(counts, field.name)) { - 0 => if (default_value(field)) |default| { - @field(result, field.name) = default; + inline for ( + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |field_name, FieldType, default_ptr| { + const flag = flag_name(field_name); + switch (@field(counts, field_name)) { + 0 => if (defaultValue(FieldType, default_ptr)) |default| { + @field(result, field_name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -268,15 +294,19 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } } - if (@hasField(Flags, "positional")) { - assert(counts.positional <= positional_fields.len); - inline for (positional_fields, 0..) |positional_field, positional_index| { + if (positional_infos) |pos_infos| { + assert(counts.positional <= pos_infos.field_names.len); + inline for ( + 0.., + pos_infos.field_names, + pos_infos.field_types, + pos_infos.field_attrs, + ) |positional_index, name, PosFieldType, attr| { if (positional_index >= counts.positional) { - const flag = comptime flag_name_positional(positional_field); - if (default_value(positional_field)) |default| { - @field(result.positional, positional_field.name) = default; + if (attr.defaultValue(PosFieldType)) |default| { + @field(result.positional, name) = default; } else { - fatal("{s}: argument is required", .{flag}); + fatal("<{s}>: argument is required", .{name}); } } } @@ -291,8 +321,8 @@ fn assert_valid_value_type(comptime T: type) void { if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; - assert(info.is_exhaustive); - assert(info.fields.len >= 2); + assert(info.mode == .exhaustive); + assert(info.field_names.len >= 2); return; } @@ -441,7 +471,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { + _ = std.math.mul(u64, amount, @backingInt(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -452,7 +482,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @intFromEnum(size.unit), + @backingInt(size.unit), ) catch unreachable; } @@ -534,7 +564,7 @@ fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).@"enum".is_exhaustive); + comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -544,53 +574,44 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_count = std.meta.fields(E).len; + const field_names = std.meta.fieldNames(E); + const field_count = field_names.len; assert(field_count >= 2); var result: []const u8 = ""; - for (std.meta.fields(E), 0..) |field, field_index| { + for (field_names, 0..) |field_name, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field.name ++ "'"; + result = result ++ separator ++ "'" ++ field_name ++ "'"; } return result; } } -pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field.name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { - result = result ++ field.name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field.name[index..]; - break :blk result; - }; +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. +pub fn flag_name(comptime field_name: []const u8) []const u8 { + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; - try std.testing.expectEqualStrings(flag_name(field), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } -fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - return "<" ++ field.name ++ ">"; +fn flag_name_positional(comptime field_name: []const u8) []const u8 { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + return "<" ++ field_name ++ ">"; } -/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. -pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value_ptr) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; +pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { + const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); + return dp.*; } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index ec3a0343..a62d4f7d 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -5,6 +5,16 @@ pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } +/// Properly format a slice of []const u8. +pub fn strings(strs: []const []const u8) FmtStrings { + return .{ .strings = strs }; +} + +/// Properly format a slice of [*:0]const u8. +pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { + return .{ .strings = strs }; +} + fn FmtSlice(T: type) type { return struct { slice: []const T, @@ -31,21 +41,58 @@ fn FmtSlice(T: type) type { }; } +pub const FmtStrings = struct { + strings: []const []const u8, + + pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); + } + + pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return try formatIntSlice(f.strings, n, 1, writer); + } +}; + +pub const FmtZeroStrings = struct { + strings: []const [*:0]const u8, + + pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); + } +}; + pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; - return writer.printFloat(x, spec); + switch (@typeInfo(@TypeOf(x))) { + .float => try writer.printFloat(x, spec), + // Handle F4.toF32() being a [2]f32 + .array => inline for (0.., x) |i, xi| { + try writer.printFloat(xi, spec); + if (i < x.len - 1) try writer.writeByte(','); + }, + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), + } } pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - switch (@typeInfo(@TypeOf(value))) { - .int => {}, + return switch (@typeInfo(@TypeOf(value))) { + .int => try writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }), + .vector => { + const vector = @typeInfo(@TypeOf(value)).vector; + const array: [vector.len]vector.child = value; + for (&array, 0..) |elem, i| { + try writer.printInt(elem, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); + if (i < array.len - 1) { + try writer.writeAll(", "); + } + } + }, else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), - } - return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); + }; } pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { @@ -60,10 +107,24 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); } +pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(value); +} + +pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(std.mem.span(value)); +} + pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); - const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; + const fmt = switch (@typeInfo(T)) { + .@"struct", .@"union", .@"enum", .@"opaque" => if (@hasDecl(T, "formatNumber")) + "{d}" + else + "{f}", + else => "{f}", + }; const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; @@ -161,30 +222,18 @@ fn printValue( switch (@typeInfo(T)) { .@"struct" => |info| { try w.writeAll(".{ "); - inline for (info.fields, 0..) |f, i| { + inline for (info.field_names, 0..) |field_name, i| { if (i > 0) try w.writeAll(", "); if (!info.is_tuple) { try w.writeByte('.'); - try w.writeAll(f.name); + try w.writeAll(field_name); try w.writeAll(" = "); } - try printValue(w, options, @field(value, f.name), max_depth - 1); + try printValue(w, options, @field(value, field_name), max_depth - 1); } try w.writeAll(" }"); }, inline else => try w.printValue("any", options, value, max_depth - 1), } } - -pub const FormatDuration = struct { - inner: std.Io.Duration, - - pub fn formatDuration(self: FormatDuration, w: *std.Io.Writer) std.Io.Writer.Error!void { - try w.printDurationSigned(@intCast(self.inner.toNanoseconds())); - } -}; - -pub fn fmtDuration(ts: std.Io.Duration) FormatDuration { - return .{ .inner = ts }; -} diff --git a/stdx/json.zig b/stdx/json.zig index c99cfb00..c1f71a4a 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -17,22 +17,23 @@ const ParseFromValueError = std.json.ParseFromValueError; /// /// ```zig /// const Answer = union { -/// number: i32, -/// numbers: []const i32, +/// number: i32, +/// numbers: []const i32, +/// +/// const Helpers = stdx.json.UnionHelpers(@This()); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; /// }; /// /// const Message = struct { /// question: []const u8; -/// answer: stdx.json.Union(Answer); +/// answer: Answer; /// } /// ``` -pub fn Union(comptime T: type) type { +pub fn UnionHelpers(comptime T: type) type { return struct { - const Self = @This(); - - value: T, - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -40,21 +41,23 @@ pub fn Union(comptime T: type) type { ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { - inline for (std.meta.fields(T)) |field| { - switch (field.type) { - bool => if (source == .bool) return .{ .value = @unionInit(T, field.name, source.bool) }, + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + switch (Field) { + bool => if (source == .bool) return @unionInit(T, field_name, source.bool), []const u8 => switch (source) { - .string => |v| return .{ .value = @unionInit(T, field.name, v) }, - .number_string => |v| return .{ .value = @unionInit(T, field.name, v) }, + .string, .number_string => |v| { + const dupe = try allocator.dupe(u8, v); + return @unionInit(T, field_name, dupe); + }, else => {}, }, - else => switch (@typeInfo(field.type)) { - .int => if (source == .integer) return .{ .value = @unionInit(T, field.name, @intCast(source.integer)) }, - .float => if (source == .float) return .{ .value = @unionInit(T, field.name, @floatCast(source.float)) }, - .@"struct" => if (source == .object) return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source.object, options)) }, + else => switch (@typeInfo(Field)) { + .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), inline else => switch (source) { - .number_string, .array => return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)) }, + .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), else => {}, }, }, @@ -62,6 +65,36 @@ pub fn Union(comptime T: type) type { } return error.UnknownField; } + + pub fn jsonStringify(self: T, jw: anytype) !void { + return switch (self) { + inline else => |v| jw.write(v), + }; + } + }; +} + +/// Deprecated: prefer importing `stdx.json.UnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.Union`. +pub fn Union(comptime T: type) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = UnionHelpers(T); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } }; } @@ -81,19 +114,31 @@ pub fn Union(comptime T: type) type { /// /// ```zig /// const Entry = union { -/// faq: struct { question: []const u8, answer: u32 }, -/// address: struct { city: []const u8, zipcode: []const u8 }, +/// faq: struct { question: []const u8, answer: u32 }, +/// address: struct { city: []const u8, zipcode: []const u8 }, +/// +/// const Helpers = stdx.json.TaggedUnionHelpers(@This(), "type"); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; /// }; /// -/// const Message = []const stdx.json.TaggedUnion(Entry, "type"); +/// const Message = []const Entry; /// ``` -pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { +pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) type { + comptime { + if (@typeInfo(T) != .@"union") { + @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); + } + for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { + if (@typeInfo(Field) != .@"struct") { + @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); + } + } + } + return struct { - const Self = @This(); - - value: T, - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -101,27 +146,66 @@ pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { errdefer std.log.warn("failed to parse: {} as {s}", .{ source, @typeName(T) }); if (source != .object) return error.UnexpectedToken; - const o = source.object; - const tag = o.get(tag_name) orelse return error.MissingField; - for (o.keys(), o.values()) |k, v| { - std.log.warn("object['{s}'] = {}", .{ k, v }); - } + var o = source.object; + const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; if (tag != .string) return error.LengthMismatch; - inline for (std.meta.fields(T)) |field| { - if (std.mem.eql(u8, field.name, tag.string)) { - const inner_source = o.get(field.name) orelse return error.MissingField; - const inner: field.type = std.json.innerParseFromValue(field.type, allocator, inner_source, options) catch |err| { - std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + if (std.mem.eql(u8, field_name, tag.string)) { + const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); return err; }; - return .{ .value = @unionInit(T, field.name, inner) }; + return @unionInit(T, field_name, inner); } } return error.InvalidEnumTag; } + + pub fn jsonStringify(self: T, jw: anytype) !void { + try jw.beginObject(); + switch (self) { + inline else => |v, tag| { + // emit a field containing the enum tag. + try jw.objectField(tag_name); + try jw.write(@tagName(tag)); + switch (@typeInfo(@TypeOf(v))) { + .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { + try jw.objectField(name); + try jw.write(@field(v, name)); + }, + else => unreachable, // Would have failed with comptime check. + } + }, + } + try jw.endObject(); + } + }; +} + +/// Deprecated: prefer importing `stdx.json.TaggedUnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.TaggedUnion`. +pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = TaggedUnionHelpers(T, tag_name); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } }; } @@ -138,16 +222,155 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { return .{ .value = (try std.json.innerParseFromValue(?T, allocator, source, options)) orelse default_value }; } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + try jw.write(self.value); + } }; } pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { - inline for (@typeInfo(T).Struct.fields) |field| { - if (field.default_value) |default_ptr| { - if (@field(r, field.name) == null) { - const default = @as(*align(1) const field.type, @ptrCast(default_ptr)).*; - @field(r, field.name) = default; + const info = @typeInfo(T).@"struct"; + inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.defaultValue(Field)) |default| { + if (@field(r, field_name) == null) { + @field(r, field_name) = default; } } } } + +test "union" { + const Answer = union(enum) { + number: i32, + numbers: []const i32, + flag: bool, + ratio: f64, + text: []const u8, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + + const Message = struct { + question: []const u8, + answer: Answer, + }; + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const number_case = "{\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_number = try std.json.parseFromSliceLeaky(Message, allocator, number_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_number.question); + try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.number); + try std.testing.expectFmt(number_case, "{f}", .{std.json.fmt(parsed_number, .{})}); + + const numbers_case = "{\"question\":\"Count to three.\",\"answer\":[1,2,3]}"; + const parsed_numbers = try std.json.parseFromSliceLeaky(Message, allocator, numbers_case, .{}); + try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.numbers.len); + try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.numbers[0]); + try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.numbers[1]); + try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.numbers[2]); + try std.testing.expectFmt(numbers_case, "{f}", .{std.json.fmt(parsed_numbers, .{})}); + + const flag_case = "{\"question\":\"Are you ready?\",\"answer\":true}"; + const parsed_flag = try std.json.parseFromSliceLeaky(Message, allocator, flag_case, .{}); + try std.testing.expectEqual(true, parsed_flag.answer.flag); + try std.testing.expectFmt(flag_case, "{f}", .{std.json.fmt(parsed_flag, .{})}); + + const ratio_case = "{\"question\":\"What is pi?\",\"answer\":3.5}"; + const parsed_ratio = try std.json.parseFromSliceLeaky(Message, allocator, ratio_case, .{}); + try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.ratio); + try std.testing.expectFmt(ratio_case, "{f}", .{std.json.fmt(parsed_ratio, .{})}); + + const text_case = "{\"question\":\"Your name?\",\"answer\":\"zml\"}"; + const parsed_text = try std.json.parseFromSliceLeaky(Message, allocator, text_case, .{}); + try std.testing.expectEqualStrings("zml", parsed_text.answer.text); + try std.testing.expectFmt(text_case, "{f}", .{std.json.fmt(parsed_text, .{})}); + + const null_value = try std.json.parseFromSliceLeaky(std.json.Value, allocator, "null", .{}); + try std.testing.expectError(error.UnknownField, Answer.jsonParseFromValue(allocator, null_value, .{})); +} + +test "strings in union can be freed" { + const Answer = union(enum) { + string: []const u8, + number: u32, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + const Struct = struct { + answer1: Answer, + answer2: Answer, + }; + const struct_json = + \\{"answer1":"I'm 29","answer2": 1e29} + ; + var parsed_value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, struct_json, .{ .parse_numbers = false }); + defer parsed_value.deinit(); + + const parsed = try std.json.parseFromValueLeaky(Struct, std.testing.allocator, parsed_value.value, .{}); + + try std.testing.expect(parsed.answer1 == .string); + try std.testing.expectEqualStrings("I'm 29", parsed.answer1.string); + std.testing.allocator.free(parsed.answer1.string); + + try std.testing.expect(parsed.answer2 == .string); + try std.testing.expectEqualStrings("1e29", parsed.answer2.string); + std.testing.allocator.free(parsed.answer2.string); +} + +test "tagged union" { + const Message = union(enum) { + faq: struct { + question: []const u8, + answer: u32, + }, + address: struct { + city: []const u8, + zipcode: []const u8, + }, + + const Helpers = TaggedUnionHelpers(@This(), "type"); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const faq_case = "{\"type\":\"faq\",\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_faq = try std.json.parseFromSliceLeaky(Message, allocator, faq_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_faq.faq.question); + try std.testing.expectEqual(@as(u32, 5), parsed_faq.faq.answer); + try std.testing.expectFmt(faq_case, "{f}", .{std.json.fmt(parsed_faq, .{})}); + + const address_case = "{\"type\":\"address\",\"city\":\"NYC\",\"zipcode\":\"49130\"}"; + const parsed_address = try std.json.parseFromSliceLeaky(Message, allocator, address_case, .{}); + try std.testing.expectEqualStrings("NYC", parsed_address.address.city); + try std.testing.expectEqualStrings("49130", parsed_address.address.zipcode); + try std.testing.expectFmt(address_case, "{f}", .{std.json.fmt(parsed_address, .{})}); +} + +test "never null" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const non_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "3", .{}); + try std.testing.expectEqual(@as(i32, 3), non_null.value); + try std.testing.expectFmt("3", "{f}", .{std.json.fmt(non_null, .{})}); + + const was_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "null", .{}); + try std.testing.expectEqual(@as(i32, 10), was_null.value); + try std.testing.expectFmt("10", "{f}", .{std.json.fmt(was_null, .{})}); +} diff --git a/stdx/meta.zig b/stdx/meta.zig index 9a096122..9b1c05af 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -2,8 +2,6 @@ const std = @import("std"); const debug = @import("debug.zig"); const compileError = debug.compileError; -pub const FnSignature = @import("signature.zig").FnSignature; -pub const Signature = @import("signature.zig").Signature; pub fn isStruct(comptime T: type) bool { return switch (@typeInfo(T)) { @@ -22,8 +20,8 @@ pub fn isTuple(comptime T: type) bool { pub fn isStructOf(comptime T: type, comptime Elem: type) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (field.type != Elem) { + inline for (info.field_types) |FieldType| { + if (FieldType != Elem) { break :blk false; } } @@ -36,8 +34,8 @@ pub fn isStructOf(comptime T: type, comptime Elem: type) bool { pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.fields) |field| { - if (f(field.type) == false) { + inline for (info.field_types) |FieldType| { + if (f(FieldType) == false) { break :blk false; } } @@ -123,9 +121,9 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - const fields = std.meta.fields(T); + const all_field_types = std.meta.fieldTypes(T); const start_ = start orelse 0; - const end_ = end orelse fields.len; + const end_ = end orelse all_field_types.len; if (start_ == end_) { return @Tuple(&.{}); @@ -133,49 +131,80 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize var field_types: [end_ - start_]type = undefined; inline for (start_..end_, 0..) |i, j| { - field_types[j] = fields[i].type; + field_types[j] = all_field_types[i]; } return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - return @typeInfo(@TypeOf(func)).@"fn".params[n].type orelse @compileError("anytype is not supported"); + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + if (n >= params.len) { + @compileError("param doesn't exist in func"); + } + return params[n] orelse @compileError("anytype is not supported"); } -pub fn FnArgs(comptime func: anytype) type { - debug.assertComptime(!@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgs expects non generic function, got: {}", .{@TypeOf(func)}); - return FnSignature(func, null).ArgsT; -} - -pub fn FnArgsWithHint(comptime func: anytype, ArgsT: type) type { - debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgsWithHint expects a generic function, got: {}", .{@TypeOf(func)}); - return FnSignature(func, ArgsT).ArgsT; -} - -pub fn FnResult(comptime func: anytype) type { +pub fn FnReturn(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } -pub fn Head(Tuple: type) type { +pub fn FnReturnPayload(comptime func: anytype) type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.payload, + else => rt, + }; +} + +pub fn FnReturnErrorSet(comptime func: anytype) ?type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.error_set, + else => null, + }; +} + +pub fn Head(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - return struct_info.fields[0].type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.field_types[0]; }, else => @compileError("Head works on tuple type"), }; } -pub fn Tail(Tuple: type) type { +pub fn Tail(comptime Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); - var types: [struct_info.fields.len - 1]type = undefined; - for (struct_info.fields[1..], &types) |field, *type_| { - type_.* = field.type; + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.field_types.len - 1]type = undefined; + for (struct_info.field_types[1..], &types) |FieldType, *type_| { + type_.* = FieldType; } return @Tuple(&types); }, else => @compileError("Tail works on tuple type"), }; } + +pub fn fnName(comptime func: anytype) []const u8 { + const T = @TypeOf(func); + std.debug.assert(@typeInfo(T) == .@"fn"); + const wrapper_name = @typeName(Wrapper(func)); + + var it = std.mem.splitSequence(u8, wrapper_name, ".Wrapper((function '"); + _ = it.first(); + const rest = it.next().?; + return rest[0 .. rest.len - 3]; +} + +fn Wrapper(comptime arg: anytype) type { + return struct { + const stuff = arg; + }; +} + +test fnName { + try std.testing.expectEqualStrings("Tail", fnName(Tail)); +} diff --git a/stdx/process.zig b/stdx/process.zig index 434a5bf7..91cf5586 100644 --- a/stdx/process.zig +++ b/stdx/process.zig @@ -15,10 +15,10 @@ fn selfSharedObjectPathImpl(addr: usize) []const u8 { return std.mem.span(info.dli_fname); } -pub fn selfSharedObjectPath() []const u8 { +pub noinline fn selfSharedObjectPath() []const u8 { return selfSharedObjectPathImpl(@returnAddress()); } -pub fn selfSharedObjectDirPath() []const u8 { +pub noinline fn selfSharedObjectDirPath() []const u8 { return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; } diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig index ee115ce2..567534be 100644 --- a/stdx/segmented_list.zig +++ b/stdx/segmented_list.zig @@ -107,7 +107,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; fn AtType(comptime SelfType: type) type { - if (@typeInfo(SelfType).pointer.is_const) { + if (@typeInfo(SelfType).pointer.attrs.@"const") { return *const T; } else { return *T; diff --git a/stdx/signature.zig b/stdx/signature.zig deleted file mode 100644 index 955a1887..00000000 --- a/stdx/signature.zig +++ /dev/null @@ -1,59 +0,0 @@ -const std = @import("std"); - -const compileError = @import("debug.zig").compileError; - -pub fn ArgsTuple(comptime funcT: anytype, comptime ArgsT: ?type) type { - const params = @typeInfo(funcT).@"fn".params; - if (params.len == 0) { - return @TypeOf(.{}); - } - - if (@typeInfo(funcT).@"fn".is_generic == false) { - return std.meta.ArgsTuple(funcT); - } - - const args = std.meta.fields(ArgsT orelse @compileError("generic function requires an explicit ArgsTuple")); - var tuple_types: [params.len]type = undefined; - if (params.len != args.len) { - compileError("function {} expected {} args, got {}", .{ funcT, params.len, args.len }); - } - inline for (params, args, 0..) |param, arg, i| { - if (param.type == null) { - tuple_types[i] = arg.type; - continue; - } - tuple_types[i] = param.type.?; - } - - return @Tuple(tuple_types); -} - -pub const Signature = struct { - Func: type, - FuncT: type, - ArgsT: type, - ReturnT: type, - ReturnPayloadT: type, - ReturnErrorSet: ?type, -}; - -pub fn FnSignature(comptime func: anytype, comptime argsT_: ?type) Signature { - const argsT = ArgsTuple(@TypeOf(func), argsT_); - const return_type = @TypeOf(@call(.auto, func, @as(argsT, undefined))); - return Signature{ - .Func = struct { - pub const Value = func; - }, - .FuncT = @TypeOf(func), - .ArgsT = argsT, - .ReturnT = return_type, - .ReturnPayloadT = switch (@typeInfo(return_type)) { - .error_union => |u| u.payload, - else => return_type, - }, - .ReturnErrorSet = switch (@typeInfo(return_type)) { - .error_union => |u| u.error_set, - else => null, - }, - }; -} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 5ccf1cb4..1b19e638 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; +pub const cpu = @import("cpu.zig"); pub const crypto = @import("crypto.zig"); pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); @@ -27,13 +28,22 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { return storage[0..len]; } -pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; +pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; + +pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { + var a: std.heap.ArenaAllocator = .init(parent); + + _ = try a.allocator().alloc(u8, initial_capacity); + std.debug.assert(a.state.used_list.?.end_index == initial_capacity); + a.state.used_list.?.end_index = 0; + return a; +} pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); - var set: CPUSet = .initEmpty(); + var set: CPUSet = .empty; set.set(core_id); std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } @@ -76,3 +86,60 @@ pub fn Once(comptime f: anytype) type { } }; } + +/// Packed storage of sub-byte values +/// +/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, +/// But that's not true anymore. +pub fn Packed(T: type) type { + return switch (@bitSizeOf(T)) { + 2 => packed struct(u8) { + x: T, + y: T, + z: T, + w: T, + + const P = @This(); + pub fn pack(x: T, y: T, z: T, w: T) P { + return .{ .x = x, .y = y, .z = z, .w = w }; + } + + pub fn unpack(p: P) [4]T { + return .{ p.x, p.y, p.z, p.w }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + 4 => packed struct(u8) { + x: T, + y: T, + + const P = @This(); + pub fn pack(x: T, y: T) P { + return .{ .x = x, .y = y }; + } + + pub fn unpack(p: P) [2]T { + return .{ p.x, p.y }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), + }; +} + +fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + for (0.., values) |i, val| { + if (i > 0) try w.writeByte(','); + switch (@typeInfo(T)) { + .@"struct" => try val.formatNumber(w, n), + .int => try fmt.formatInt(val, n, w), + else => try w.print("{}", .{val}), + } + } +} diff --git a/third_party/abseil-cpp/bmi2intrin.patch b/third_party/abseil-cpp/bmi2intrin.patch new file mode 100644 index 00000000..8273731b --- /dev/null +++ b/third_party/abseil-cpp/bmi2intrin.patch @@ -0,0 +1,13 @@ +diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h +index 0d05ad7..85ed32d 100644 +--- a/absl/container/internal/raw_hash_set.h ++++ b/absl/container/internal/raw_hash_set.h +@@ -226,7 +226,7 @@ static_assert(std::numeric_limits::is_iec559, + #endif + + #ifdef __BMI2__ +-#include ++#include + #endif // __BMI2__ + + #if defined(__SSE2__) || defined(__aarch64__) || \ \ No newline at end of file diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index 3e3c62bd..a5de1d98 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -1,54 +1,89 @@ -load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") -write_file( - name = "system_defaults_zig", - out = "system_defaults.zig", - content = ["""\ -pub const enable_linker_build_id: bool = false; -pub const linker: []const u8 = ""; -pub const sysroot: []const u8 = ""; -pub const gcc_install_prefix: []const u8 = ""; -pub const rtlib: []const u8 = ""; -pub const unwindlib: []const u8 = ""; -"""], +zig_option( + name = "enable_linker_build_id", + bool = False, ) -zig_library( +zig_option( + name = "linker", + string = "", +) + +zig_option( + name = "sysroot", + string = "", +) + +zig_option( + name = "gcc_install_prefix", + string = "", +) + +zig_option( + name = "rtlib", + string = "", +) + +zig_option( + name = "unwindlib", + string = "", +) + +zig_options( name = "system_defaults", - main = ":system_defaults_zig", + settings = [ + ":enable_linker_build_id", + ":linker", + ":sysroot", + ":gcc_install_prefix", + ":rtlib", + ":unwindlib", + ], ) -write_file( - name = "aro_options_zig", - out = "aro_options.zig", - content = ["""\ -pub const enable_tracy: bool = false; -pub const enable_tracy_callstack: bool = false; -pub const enable_tracy_allocation: bool = false; -pub const debug_allocations: bool = false; -pub const version_str: []const u8 = ""; -"""], +zig_option( + name = "enable_tracy", + bool = False, ) -zig_library( +zig_option( + name = "enable_tracy_callstack", + bool = False, +) + +zig_option( + name = "enable_tracy_allocation", + bool = False, +) + +zig_option( + name = "debug_allocations", + bool = False, +) + +zig_option( + name = "version_str", + string = "", +) + +zig_options( name = "build_options", - main = ":aro_options_zig", -) - -zig_library( - name = "zig", - srcs = glob(["deps/zig/**/*.zig"]), - main = "deps/zig/lib.zig", + settings = [ + ":enable_tracy", + ":enable_tracy_callstack", + ":enable_tracy_allocation", + ":debug_allocations", + ":version_str", + ], ) zig_library( name = "backend", main = "src/backend.zig", - deps = [ - ":build_options", - ":zig", - ], + deps = [":build_options"], ) zig_library( @@ -83,7 +118,6 @@ zig_binary( ":assembly_backend", ":build_options", ], - zigopts = ["-fm"], ) zig_binary( @@ -92,12 +126,15 @@ zig_binary( ) [ - genrule( + run_binary( name = "def_{}".format(name), srcs = [name], outs = ["{}.zig".format(name)], - cmd = """$(location :generate-def) $< $@""", - tools = [":generate-def"], + args = [ + "$(location {})".format(name), + "$(location {}.zig)".format(name), + ], + tool = ":generate-def", ) for name in glob(["src/aro/**/*.def"]) ] diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 510445d1..15ff6d7f 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,9 +1,10 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + # https://github.com/zml/arocc/releases/tag/0.17.0 + git_repository( name = "arocc", - remote = "https://github.com/zml/arocc.git", - commit = "73acf3aa8164cc8fee16b0f79036b92b07dab053", - build_file = "//third_party/arocc:arocc.bazel", + remote = "https://github.com/zml/arocc", + commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", + build_file = Label("//third_party/arocc:arocc.bazel"), ) diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel b/third_party/cloud_accelerator_diagnostics/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel rename to third_party/cloud_accelerator_diagnostics/BUILD.bazel diff --git a/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel new file mode 100644 index 00000000..563999d2 --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel @@ -0,0 +1,34 @@ +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") + +proto_library( + name = "tpu_telemetry_proto", + srcs = ["tpu_telemetry.proto"], + import_prefix = "tpu_info/proto", + visibility = ["//visibility:public"], +) + +proto_library( + name = "tpu_metric_service_proto", + srcs = ["tpu_metric_service.proto"], + deps = [ + ":tpu_telemetry_proto", + "@com_google_protobuf//:timestamp_proto", + ], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "tpu_metric_service_cc_proto", + deps = [":tpu_metric_service_proto"], + visibility = ["//visibility:public"], +) + +cc_grpc_library( + name = "tpu_metric_service_cc_grpc", + srcs = [":tpu_metric_service_proto"], + grpc_only = True, + deps = [":tpu_metric_service_cc_proto"], + visibility = ["//visibility:public"], +) diff --git a/third_party/cloud_accelerator_diagnostics/repo.bzl b/third_party/cloud_accelerator_diagnostics/repo.bzl new file mode 100644 index 00000000..661e45ec --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/repo.bzl @@ -0,0 +1,29 @@ +_COMMIT = "6eccaf47364904d1c55209dd8795e3bf9cac21e1" +_URL_PREFIX = "https://raw.githubusercontent.com/AI-Hypercomputer/cloud-accelerator-diagnostics/" + _COMMIT + "/tpu_info/tpu_info/proto/" + +_PROTOS = { + "tpu_metric_service.proto": "a89d80938c75177ff45fb4c6130f5d71bc5164df473a77948e31b7dedd96a1bb", + "tpu_telemetry.proto": "4ad6096e40de5ba00b6c7badea1218c801069b82be829ec56dc0ce693b5a2505", +} + +def _impl(ctx): + for output, sha256 in _PROTOS.items(): + ctx.download( + url = _URL_PREFIX + output, + sha256 = sha256, + output = output, + ) + ctx.template("BUILD.bazel", ctx.attr.build_file) + +_cloud_accelerator_diagnostics = repository_rule( + implementation = _impl, + attrs = { + "build_file": attr.label(mandatory = True), + }, +) + +def repo(): + _cloud_accelerator_diagnostics( + name = "cloud_accelerator_diagnostics", + build_file = "//third_party/cloud_accelerator_diagnostics:cloud_accelerator_diagnostics.BUILD.bazel", + ) diff --git a/third_party/com_google_sentencepiece/BUILD.bazel b/third_party/com_google_sentencepiece/BUILD.bazel index 53e12d3a..598dce0b 100644 --- a/third_party/com_google_sentencepiece/BUILD.bazel +++ b/third_party/com_google_sentencepiece/BUILD.bazel @@ -28,12 +28,12 @@ cc_library( includes = ["."], visibility = ["//visibility:public"], deps = [ - "@abseil-cpp//absl/container:flat_hash_map", - "@abseil-cpp//absl/container:flat_hash_set", - "@abseil-cpp//absl/flags:flag", - "@abseil-cpp//absl/flags:parse", - "@abseil-cpp//absl/flags:usage", - "@abseil-cpp//absl/strings", - "@abseil-cpp//absl/strings:string_view", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/flags:usage", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", ], ) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel b/third_party/cuda_tile/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel rename to third_party/cuda_tile/BUILD.bazel diff --git a/third_party/cuda_tile/cuda_tile.bazel b/third_party/cuda_tile/cuda_tile.bazel new file mode 100644 index 00000000..bfa768e9 --- /dev/null +++ b/third_party/cuda_tile/cuda_tile.bazel @@ -0,0 +1,450 @@ +# XLA's third_party/cuda_tile/cuda_tile.BUILD, plus the C API library it does +# not build (cuda_tile_capi at the bottom). +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_binary( + name = "cuda-tile-tblgen", + srcs = glob([ + "tools/cuda-tile-tblgen/*.cpp", + "tools/cuda-tile-tblgen/*.h", + ]), + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//llvm:TableGen", + "@llvm-project//mlir:MlirTableGenMain", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TableGen", + ], +) + +exports_files(["LICENSE"]) + +td_library( + name = "CudaTileTdFiles", + srcs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:ControlFlowInterfacesTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:ViewLikeInterfaceTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileDialectIncGen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Dialect.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileInterfacesIncGen", + tbl_outs = [ + ( + ["-gen-attr-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.h.inc", + ), + ( + ["-gen-attr-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.cpp.inc", + ), + ( + ["-gen-type-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Interfaces.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileTypesIncGen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.cpp.inc", + ), + ( + [ + "-gen-type-constraint-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.h.inc", + ), + ( + [ + "-gen-type-constraint-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Types.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileAttrDefsIncGen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.cpp.inc", + ), + ( + [ + "-gen-enum-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.h.inc", + ), + ( + [ + "-gen-enum-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileAttrCodeGenIncGen", + tbl_outs = [ + ( + ["-gen-enum-attr-list"], + "include/cuda_tile/Dialect/CudaTile/IR/CudaTileEnumAttrs.inc", + ), + ( + ["-gen-gpu-arch-def"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchitectures.inc", + ), + ( + ["-gen-gpu-arch-constants"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchConstants.inc", + ), + ( + ["-gen-hint-key-accessors"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyAccessors.inc", + ), + ( + ["-gen-hint-key-impl"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyImpl.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileRemarksIncGen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.h.inc", + ), + ( + ["-gen-enum-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Remarks.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsIncGen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsCanonicalizationIncGen", + tbl_outs = [ + ( + ["-gen-rewriters"], + "lib/Dialect/CudaTile/IR/OpsCanonicalization.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "lib/Dialect/CudaTile/IR/OpsCanonicalization.td", + deps = [ + ":CudaTileTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + ], +) + +cc_library( + name = "CudaTileDialect", + srcs = glob( + ["lib/Dialect/CudaTile/IR/*.cpp"], + exclude = ["lib/Dialect/CudaTile/IR/CudaTileTesting.cpp"], + ), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.h"]), + includes = [ + "include", + "lib/Dialect/CudaTile/IR", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileAttrCodeGenIncGen", + ":CudaTileAttrDefsIncGen", + ":CudaTileDialectIncGen", + ":CudaTileInterfacesIncGen", + ":CudaTileOpsCanonicalizationIncGen", + ":CudaTileOpsIncGen", + ":CudaTileRemarksIncGen", + ":CudaTileTypesIncGen", + "@llvm-project//llvm:Core", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:ControlFlowInterfaces", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:ViewLikeInterface", + ], +) + +td_library( + name = "CudaTileTransformsTdFiles", + srcs = ["include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td"], + includes = ["include"], + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileTransformsIncGen", + tbl_outs = [ + ( + [ + "-gen-pass-decls", + "-name=CudaTile", + ], + "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.h.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td", + deps = [":CudaTileTransformsTdFiles"], +) + +cc_library( + name = "CudaTileTransforms", + srcs = glob(["lib/Dialect/CudaTile/Transforms/*.cpp"]), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/Transforms/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileDialect", + ":CudaTileTransformsIncGen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:TransformUtils", + ], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpsIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-bytecode"], + "include/cuda_tile/Bytecode/Writer/Bytecode.inc", + ), + ( + ["-gen-cuda-tile-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/BytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpcodesIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-opcodes"], + "include/cuda_tile/Bytecode/Common/StaticOpcodes.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeTypeIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-type-bytecode"], + "include/cuda_tile/Bytecode/Writer/TypeBytecode.inc", + ), + ( + ["-gen-cuda-tile-type-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/TypeBytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeTypeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeAttrIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-attr-bytecode"], + "include/cuda_tile/Bytecode/Writer/AttrBytecode.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeAttrOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +cc_library( + name = "CudaTileBytecode", + srcs = glob([ + "lib/Bytecode/**/*.cpp", + "lib/Bytecode/**/*.h", + ]), + hdrs = glob([ + "include/cuda_tile/Bytecode/**/*.h", + ]), + includes = [ + "include", + "include/cuda_tile/Bytecode/Common", + "include/cuda_tile/Bytecode/Reader", + "include/cuda_tile/Bytecode/Writer", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecodeAttrIncGen", + ":CudaTileBytecodeOpcodesIncGen", + ":CudaTileBytecodeOpsIncGen", + ":CudaTileBytecodeTypeIncGen", + ":CudaTileDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TranslateLib", + ], +) + +cc_library( + name = "CudaTileOptimizer", + srcs = ["lib/Dialect/CudaTile/Optimizer/CudaTileOptimizer.cpp"], + hdrs = ["include/cuda_tile/Dialect/CudaTile/Optimizer/CudaTileOptimizer.h"], + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileTransforms", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +# The MLIR C API for the dialect: types, enum attributes, bytecode, passes, and +# the dialect handle `mlirGetDialectHandle__cuda_tile__` that ZML resolves at +# comptime. Mirrors lib/CAPI/CMakeLists.txt + lib/CAPI/Dialect/CMakeLists.txt. +cc_library( + name = "cuda_tile_capi", + srcs = [ + "lib/CAPI/Dialect/CudaTileDialect.cpp", + "lib/CAPI/Dialect/CudaTileOptimizer.cpp", + "lib/CAPI/Registration.cpp", + ], + hdrs = glob(["include/cuda_tile-c/**/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileOptimizer", + ":CudaTileTransforms", + "@llvm-project//mlir:CAPIIR", + ], + # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static + # initializer that the linker would otherwise dead-strip. + alwayslink = True, +) diff --git a/third_party/cuda_tile/repo.bzl b/third_party/cuda_tile/repo.bzl new file mode 100644 index 00000000..79076c51 --- /dev/null +++ b/third_party/cuda_tile/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +# v13.4.0 with upstream LLVM compatibility fixes +def repo(): + git_repository( + name = "cuda_tile", + remote = "https://github.com/NVIDIA/cuda-tile.git", + commit = "5b3fa4c0dd04d90edeb935a1887c20bf14a86ebe", + build_file = "//third_party/cuda_tile:cuda_tile.bazel", + ) diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel b/third_party/cute_ir/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel rename to third_party/cute_ir/BUILD.bazel diff --git a/third_party/cute_ir/cute_ir.bazel b/third_party/cute_ir/cute_ir.bazel new file mode 100644 index 00000000..0c0aed19 --- /dev/null +++ b/third_party/cute_ir/cute_ir.bazel @@ -0,0 +1,104 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # BSD-3-Clause + +# The CuTe dialect of the cutlass_compiler/ tree, as released. The tree that +# builds it (//mlir/dialects/cute_ir) adds the DSL compiler's own attributes, +# types and operations and compiles these sources with them. +CUTE_IR_INCLUDE = "cute_ir/include" + +exports_files([ + "LICENSE.txt", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "cute_ir/lib/Dialect/IR/CuteDialect.cpp", +]) + + +# cutegen, the CuTe layout algebra, with the `?{div=N}` leaf added by +# cutegen_dynamic_divisibility.patch: a dynamic extent the DSL knows to be a +# multiple of N, which the release's cutegen cannot read. +cc_library( + name = "cutegen", + hdrs = glob(["cutegen/include/cutegen/*.hpp"]), + includes = ["cutegen/include"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) + +td_library( + name = "cute_ir_dialect_td", + srcs = [ + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = [CUTE_IR_INCLUDE], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "cute_ir_dialect_inc_gen", + includes = [CUTE_IR_INCLUDE], + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":cute_ir_dialect_td"], +) + +gentbl_cc_library( + name = "cute_ir_type_interfaces_inc_gen", + includes = [CUTE_IR_INCLUDE], + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":cute_ir_dialect_td"], +) + +# The header alone: the attribute, type and operation class declarations it +# includes are generated by the building tree, which adds the compiler's own. +cc_library( + name = "cute_ir_dialect_header", + hdrs = [CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h"], + includes = [CUTE_IR_INCLUDE], + deps = [ + ":cute_ir_dialect_inc_gen", + ":cute_ir_type_interfaces_inc_gen", + ], +) diff --git a/third_party/cute_ir/cutegen_dynamic_divisibility.patch b/third_party/cute_ir/cutegen_dynamic_divisibility.patch new file mode 100644 index 00000000..4f479d60 --- /dev/null +++ b/third_party/cute_ir/cutegen_dynamic_divisibility.patch @@ -0,0 +1,243 @@ +--- a/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp ++++ b/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp +@@ -73,13 +73,20 @@ + * @brief Width (in bits) of the associated dynamic integer + */ + int32_t width = default_dynamic_int_bitwidth(); ++ /** ++ * @brief Known positive divisor of the associated dynamic integer; 1 when ++ * nothing is known. Spelled `div=N` in the dynamic leaf's properties. ++ */ ++ static_int_t divisibility = 1; + /// Constructor + constexpr dynamic_int_properties_t() = default; + explicit dynamic_int_properties_t(int32_t w); ++ dynamic_int_properties_t(int32_t w, static_int_t div); + /// Equality operators + bool operator==(const dynamic_int_properties_t& rhs) const; + bool operator!=(const dynamic_int_properties_t& rhs) const; + dynamic_int_properties_t with_width(int32_t w) const; ++ dynamic_int_properties_t with_divisibility(static_int_t div) const; + bool is_default() const; + bool are_compatible(static_int_t v) const; + static_int_t minimum() const; +@@ -475,11 +482,25 @@ + static std::string to_str(const dynamic_t& d) + { + std::string s("?"); +- if(d.get_width() != default_dynamic_int_bitwidth()) ++ const bool has_width = d.get_width() != default_dynamic_int_bitwidth(); ++ const bool has_div = d.get_properties().divisibility != 1; ++ if(has_width || has_div) + { + s.append("{"); +- s.append("i"); +- s.append(std::to_string(d.get_width())); ++ if(has_width) ++ { ++ s.append("i"); ++ s.append(std::to_string(d.get_width())); ++ } ++ if(has_div) ++ { ++ if(has_width) ++ { ++ s.append(" "); ++ } ++ s.append("div="); ++ s.append(std::to_string(d.get_properties().divisibility)); ++ } + s.append("}"); + } + return s; +@@ -553,6 +574,22 @@ + return; + } + } ++ else if(kw.value() == "div") ++ { ++ auto eq = parse_type(); ++ if(!eq.has_value()) ++ { ++ // Error: missing '=' after 'div' ++ return; ++ } ++ auto div = parse_type(); ++ if(!div.has_value() || div.value() <= 0) ++ { ++ // Error: no positive value for the divisor ++ return; ++ } ++ prop.divisibility = div.value(); ++ } + else + { + // Error: an unexpected keyword was found +@@ -572,6 +609,7 @@ + static void encode_value(const dynamic_t& d, TBuffer& buffer) + { + encode(buffer, d.get_width()); ++ encode(buffer, d.get_properties().divisibility); + } + }; + +@@ -586,7 +624,10 @@ + if(!w.has_value() || + (widths.end() == std::find(widths.begin(), widths.end(), w.value()))) + return std::nullopt; +- return dynamic_t(w.value()); ++ const auto div = decode(it, end); ++ if(!div.has_value() || div.value() <= 0) ++ return std::nullopt; ++ return dynamic_t(dynamic_int_properties_t(w.value(), div.value())); + } + }; + +@@ -816,45 +857,72 @@ + struct property_policy_c_t + { + using int_t = static_int_t; ++ // The divisibility a result keeps: a sum or difference is a multiple of ++ // the gcd of its operands' divisors, a product of their product, and a ++ // static operand counts as its own absolute value. ++ static static_int_t gcd_div(static_int_t a, static_int_t b) ++ { ++ a = a < 0 ? -a : a; ++ b = b < 0 ? -b : b; ++ while(b != 0) ++ { ++ static_int_t t = a % b; ++ a = b; ++ b = t; ++ } ++ return a == 0 ? 1 : a; ++ } ++ static dynamic_int_properties_t with_div(int32_t width, static_int_t div) ++ { ++ return dynamic_int_properties_t(width, div <= 0 ? 1 : div); ++ } + //-------------------------------------------------------------------------- + // resolve_add() + static arith_resolve_t resolve_add(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_add(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + static arith_resolve_t resolve_sub(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(TInt lhs, const dynamic_t& rhs) + { + auto res_width = std::max(rhs.get_width(), in_range(lhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(rhs.get_properties().divisibility, static_int_t(lhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mul() + static arith_resolve_t resolve_mul(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * rhs.get_properties().divisibility; ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> +@@ -870,7 +938,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * ++ static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mod() +@@ -939,7 +1009,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto d = lhs.get_properties().divisibility; ++ auto r = static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, d % r == 0 ? d / r : 1), res_width); + } + template , bool> = true> +@@ -1259,12 +1331,18 @@ + + inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w) : + width(w) ++{ ++} ++ ++inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w, static_int_t div) : ++ width(w), ++ divisibility(div) + { + } + + inline bool dynamic_int_properties_t::operator==(const dynamic_int_properties_t& rhs) const + { +- return (width == rhs.width); ++ return (width == rhs.width) && (divisibility == rhs.divisibility); + } + + inline bool dynamic_int_properties_t::operator!=(const dynamic_int_properties_t& rhs) const +@@ -1284,9 +1362,16 @@ + return r; + } + ++inline dynamic_int_properties_t dynamic_int_properties_t::with_divisibility(static_int_t div) const ++{ ++ dynamic_int_properties_t r = *this; ++ r.divisibility = div; ++ return r; ++} ++ + inline bool dynamic_int_properties_t::are_compatible(static_int_t v) const + { +- return in_range_of_width(v, width); ++ return in_range_of_width(v, width) && (v % divisibility == 0); + } + + inline static_int_t dynamic_int_properties_t::minimum() const +@@ -1313,7 +1398,7 @@ + + inline bool dynamic_int_properties_t::is_constrained() const + { +- return false; ++ return divisibility != 1; + } + + inline dynamic_t::dynamic_t(int32_t width_) : diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl new file mode 100644 index 00000000..6beb5df0 --- /dev/null +++ b/third_party/cute_ir/repo.bzl @@ -0,0 +1,22 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # The cutlass_compiler/ tree of the CUTLASS v4.8.0 release: the open-source + # part of the CuTe dialect that mlir/dialects/cute_ir builds on. + git_repository( + name = "cute_ir", + remote = "https://github.com/NVIDIA/cutlass.git", + commit = "098de2a652cf8f00fd70b2df54051c7eccbb855a", + build_file = Label("//third_party/cute_ir:cute_ir.bazel"), + strip_prefix = "cutlass_compiler", + patches = [ + # `cute.slice` upstream only infers tuple and layout results; the + # MoE kernels slice memrefs and coordinate tensors with explicit + # result types. + Label("//third_party/cute_ir:slice_tensor.patch"), + # The DSL spells a dynamic leaf it knows to be a multiple of N as + # `?{div=N}`; the release's cutegen reads only the width. + Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch"), + ], + patch_args = ["-p2"], + ) diff --git a/third_party/cute_ir/slice_tensor.patch b/third_party/cute_ir/slice_tensor.patch new file mode 100644 index 00000000..173a795b --- /dev/null +++ b/third_party/cute_ir/slice_tensor.patch @@ -0,0 +1,45 @@ +diff --git a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +--- a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td ++++ b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +@@ -2990,7 +2990,7 @@ + + def Cute_SliceOp : Cute_Op<"slice", [ + Pure, +- DeclareOpInterfaceMethods ++ DeclareOpInterfaceMethods + ]> { + let cuteSpecGroup = "LayoutAlgebra"; + let summary = "Slice a value, keeping modes where the coordinate is `_`"; +@@ -3154,11 +3154,11 @@ + }]; + + let arguments = (ins +- Cute_Arg:$input, ++ Cute_Arg:$input, + Cute_Arg:$coord + ); + let results = (outs +- Cute_Arg:$result ++ Cute_Arg:$result + ); + let assemblyFormat = [{ `(` operands `)` attr-dict `:` type(operands) }]; + } +diff --git a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +--- a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp ++++ b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +@@ -1327,6 +1327,15 @@ + }); + } + ++LogicalResult SliceOp::refineReturnTypes( ++ MLIRContext *, std::optional, ValueRange, DictionaryAttr, ++ PropertyRef, RegionRange, SmallVectorImpl &) { ++ // Tensor views are private CuTe-DSL types. Their result type is supplied ++ // explicitly by the front end, so no public layout-algebra refinement is ++ // required here. ++ return success(); ++} ++ + //===----------------------------------------------------------------------===// + // DiceOp (cute.dice) + //===----------------------------------------------------------------------===// diff --git a/third_party/cutlass/cutlass.bazel b/third_party/cutlass/cutlass.bazel new file mode 100644 index 00000000..e009a6d4 --- /dev/null +++ b/third_party/cutlass/cutlass.bazel @@ -0,0 +1,111 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +td_library( + name = "dialect_td", + srcs = [ + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "type_interfaces_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", + ), + ( + ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + deps = [":dialect_td"], +) + +cc_library( + name = "headers", + hdrs = [ + hdr for hdr in [ + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "include/cute_ir/Dialect/Cute/IR/CuteOps.h", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h", + ] + ], + strip_include_prefix = "include", + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":type_interfaces_inc_gen", + ], +) diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel deleted file mode 100644 index 3f32c61c..00000000 --- a/third_party/flashattn/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -filegroup( - name = "libflashattn", - visibility = ["//visibility:public"], - srcs = ["flashattn/lib/libflashattn.so"], -) - -cc_library( - name = "c", - hdrs = ["flashattn/include/capi.h"], - visibility = ["//visibility:public"], -) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl deleted file mode 100644 index 654620cc..00000000 --- a/third_party/flashattn/repo.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -def repo(): - http_archive( - name = "flashattn", - url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", - sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", - build_file = "//:third_party/flashattn/BUILD.bazel", - ) - - #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") - diff --git a/third_party/flashinfer_cutlass_moe/BUILD.bazel b/third_party/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..b2ddeede --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "flashinfer_cutlass_moe.BUILD.bazel", + "repo.bzl", +]) diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel new file mode 100644 index 00000000..780ac0d2 --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "lib_sm100", + srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], +) + +filegroup( + name = "lib_sm103", + srcs = ["lib/libflashinfer_cutlass_moe_sm103.so"], +) + +filegroup( + name = "lib_sm120", + srcs = ["lib/libflashinfer_cutlass_moe_sm120.so"], +) + +filegroup( + name = "libraries", + srcs = [ + ":lib_sm100", + ":lib_sm103", + ":lib_sm120", + ], +) + +cc_library( + name = "c", + hdrs = ["include/flashinfer_cutlass_moe.h"], + includes = ["include"], +) diff --git a/third_party/flashinfer_cutlass_moe/repo.bzl b/third_party/flashinfer_cutlass_moe/repo.bzl new file mode 100644 index 00000000..2f897c2d --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/repo.bzl @@ -0,0 +1,38 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_RELEASE = "cutlass-moe-v0.2.0" + +_ASSETS = { + "amd64": { + "sha256": "5c3a29da0fd91e2281d046eb2e467a7a40aeb24a0be8ba0487006356d45fecc4", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-amd64.tar.gz", + }, + "arm64": { + "sha256": "60b6fdbd193e11c24589add1eb206f59a3900cde4760e53719d6b2add492343e", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-arm64.tar.gz", + }, +} + +def _archive(name, arch): + asset = _ASSETS[arch] + + http_archive( + name = name, + urls = [ + asset["url"].format(release = _RELEASE), + ], + sha256 = asset["sha256"], + build_file = Label( + "//third_party/flashinfer_cutlass_moe:flashinfer_cutlass_moe.BUILD.bazel", + ), + ) + +def repo(): + _archive( + name = "flashinfer_cutlass_moe_linux_amd64", + arch = "amd64", + ) + _archive( + name = "flashinfer_cutlass_moe_linux_arm64", + arch = "arm64", + ) diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel b/third_party/flydsl/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel rename to third_party/flydsl/BUILD.bazel diff --git a/third_party/flydsl/atomic_exchange.patch b/third_party/flydsl/atomic_exchange.patch new file mode 100644 index 00000000..7de76db1 --- /dev/null +++ b/third_party/flydsl/atomic_exchange.patch @@ -0,0 +1,25 @@ +diff --git a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +--- a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +@@ -43,7 +43,8 @@ + I32EnumAttrCase<"And", 3, "and">, + I32EnumAttrCase<"Or", 4, "or">, + I32EnumAttrCase<"Inc", 5, "inc">, +- I32EnumAttrCase<"Dec", 6, "dec"> ++ I32EnumAttrCase<"Dec", 6, "dec">, ++ I32EnumAttrCase<"Exchange", 7, "exchange"> + ]>; + def Fly_AtomicOpAttr : Fly_EnumAttr {} + +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -228,6 +228,8 @@ + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::uinc_wrap); + case AtomicOp::Dec: + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::udec_wrap); ++ case AtomicOp::Exchange: ++ return LLVM::AtomicBinOp::xchg; + } + return std::nullopt; + } diff --git a/third_party/flydsl/flydsl.bazel b/third_party/flydsl/flydsl.bazel new file mode 100644 index 00000000..7c8cb7e9 --- /dev/null +++ b/third_party/flydsl/flydsl.bazel @@ -0,0 +1,423 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + default_visibility = ["//visibility:public"], + features = [ + "-parse_headers", + "-use_header_modules", + ], + licenses = ["notice"], +) + +exports_files(["LICENSE"]) + +td_library( + name = "td_files", + srcs = glob(["include/flydsl/**/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:BuiltinDialectBytecodeTdFiles", + "@llvm-project//mlir:BytecodeOpInterfaceTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:GPUOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:PassBaseTdFiles", + "@llvm-project//mlir:ROCDLOpsTdFiles", + "@llvm-project//mlir:SCFTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:VectorOpsTdFiles", + ], +) + +gentbl_cc_library( + name = "fly_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyDialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/Fly/IR/FlyDialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_interfaces_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.cpp.inc": ["--gen-attr-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.h.inc": ["--gen-attr-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.cpp.inc": ["--gen-op-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.h.inc": ["--gen-op-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.cpp.inc": ["--gen-type-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.h.inc": ["--gen-type-interface-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyInterfaces.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_types_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.cpp.inc": [ + "--gen-type-constraint-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.h.inc": [ + "--gen-type-constraint-decls", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.cpp.inc": [ + "--gen-attr-constraint-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.h.inc": [ + "--gen-attr-constraint-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyEnums.cpp.inc": ["--gen-enum-defs"], + "include/flydsl/Dialect/Fly/IR/FlyEnums.h.inc": ["--gen-enum-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyOps.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOps.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyOps.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=Fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/Passes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_int_tuple_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_layout_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_memref_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_atom_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Atom.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/Atom.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.cpp.inc": [ + "--gen-enum-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.h.inc": [ + "--gen-enum-decls", + "--typedefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Atom.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.cpp.inc": [ + "--gen-enum-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.h.inc": [ + "--gen-enum-decls", + "--attrdefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Ops.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Ops.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Ops.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_to_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Conversion/FlyToROCDL/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyToROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Conversion/FlyToROCDL/Passes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.td", + deps = [":td_files"], +) + +cc_library( + name = "FlyDialect", + srcs = [ + "lib/Dialect/Fly/IR/FlyAttrDefs.cpp", + "lib/Dialect/Fly/IR/FlyDialect.cpp", + "lib/Dialect/Fly/IR/FlyOps.cpp", + "lib/Dialect/Fly/IR/FlyTypeDefs.cpp", + "lib/Dialect/Fly/IR/FlyUniversalOps.cpp", + "lib/Dialect/Fly/Utils/IntTupleUtils.cpp", + "lib/Dialect/Fly/Utils/IntUtils.cpp", + "lib/Dialect/Fly/Utils/NormalForm.cpp", + "lib/Dialect/Fly/Utils/PointerUtils.cpp", + ], + hdrs = glob( + [ + "include/flydsl/Dialect/Fly/IR/*.h", + "include/flydsl/Dialect/Fly/Utils/*.h", + "include/flydsl/Dialect/Fly/Utils/*.inc", + ], + exclude = ["include/flydsl/Dialect/Fly/IR/FlyLLVMTranslation.h"], + ), + includes = ["include"], + deps = [ + ":fly_attrs_inc_gen", + ":fly_dialect_inc_gen", + ":fly_interfaces_inc_gen", + ":fly_ops_inc_gen", + ":fly_types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:ir_headers", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "FlyTransforms", + srcs = glob(["lib/Dialect/Fly/Transforms/*.cpp"]), + hdrs = glob(["include/flydsl/Dialect/Fly/Transforms/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_int_tuple_lowering_inc_gen", + ":fly_layout_lowering_inc_gen", + ":fly_memref_lowering_inc_gen", + ":fly_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:UBDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyROCDLDialect", + srcs = [ + "lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmAtomBuilder.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmGeometry.cpp", + "lib/Dialect/FlyROCDL/Dialect.cpp", + "lib/Dialect/FlyROCDL/GFX11/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX120X/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp", + "lib/Dialect/FlyROCDL/Ops.cpp", + "lib/Dialect/FlyROCDL/Transforms/ClusterAttr.cpp", + "lib/Dialect/FlyROCDL/Transforms/ExpandOps.cpp", + ], + hdrs = glob([ + "include/flydsl/Dialect/FlyROCDL/**/*.h", + "lib/Dialect/FlyROCDL/**/*.h", + ]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_rocdl_atom_inc_gen", + ":fly_rocdl_attrs_inc_gen", + ":fly_rocdl_dialect_inc_gen", + ":fly_rocdl_ops_inc_gen", + ":fly_rocdl_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyToROCDL", + srcs = [ + "lib/Conversion/FlyToROCDL/FastMathOps.cpp", + "lib/Conversion/FlyToROCDL/FlyToROCDL.cpp", + ], + hdrs = glob(["include/flydsl/Conversion/**/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":FlyROCDLDialect", + ":fly_to_rocdl_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:ControlFlowTransforms", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:VectorDialect", + ], +) + +# ZML addition: the per-type accessors FlyDSL's Python bindings use to derive +# tiled TV layouts and MMA tile sizes. They live under lib/Bindings/Python but +# depend only on the dialect (no nanobind). +cc_library( + name = "TiledOpTraits", + srcs = ["lib/Bindings/Python/TiledOpTraits.cpp"], + hdrs = ["lib/Bindings/Python/TiledOpTraits.h"], + includes = ["lib/Bindings/Python"], + deps = [ + ":FlyDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/third_party/flydsl/mlir_legacy_assembly.patch b/third_party/flydsl/mlir_legacy_assembly.patch new file mode 100644 index 00000000..0739e489 --- /dev/null +++ b/third_party/flydsl/mlir_legacy_assembly.patch @@ -0,0 +1,39 @@ +--- a/include/flydsl/Dialect/Fly/IR/FlyDialect.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyDialect.td +@@ -19,6 +19,11 @@ + + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; ++ ++ // ZML custom-call inputs use Fly assembly printed by a separately pinned ++ // MLIR/Fly build. Keep accepting its inherent attributes in attr-dict until ++ // the producer and consumer can migrate to prop-dict together. ++ let useStrictPropertiesInAssemblyFormat = 0; + } + + class Fly_Type traits = []> +@@ -38,7 +43,10 @@ + } + + class Fly_EnumAttr traits = []> +- : EnumAttr; ++ : EnumAttr { ++ // Keep the Fly text ABI independent of MLIR's default enum delimiters. ++ let assemblyFormat = "$value"; ++} + + class Fly_Op traits = []> + : Op { +diff --git a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +index 2b745065..875d8a69 100644 +--- a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td ++++ b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +@@ -44,7 +44,7 @@ class FlyROCDL_MakeTiledTdmAtomOp + let results = (outs AnyType:$atom, AnyType:$coordTensor); + + let assemblyFormat = [{ +- $tensor `,` $smemLayout `,` $tiler attr-dict `:` type(operands) ++ $tensor `,` $smemLayout `,` $tiler prop-dict attr-dict `:` type(operands) + }]; + } + diff --git a/third_party/flydsl/mlir_property_ref.patch b/third_party/flydsl/mlir_property_ref.patch new file mode 100644 index 00000000..145bcc1b --- /dev/null +++ b/third_party/flydsl/mlir_property_ref.patch @@ -0,0 +1,128 @@ +diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +index 80d2fb3b..7981077c 100644 +--- a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp ++++ b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +@@ -13,6 +13,7 @@ + #include "mlir/Dialect/SCF/Transforms/Patterns.h" + #include "mlir/Dialect/Vector/IR/VectorOps.h" + #include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinOps.h" + #include "mlir/IR/BuiltinTypes.h" + #include "mlir/Pass/Pass.h" + #include "mlir/Transforms/DialectConversion.h" +@@ -85,10 +86,10 @@ static unsigned getLLVMAlignment(fly::PointerType ptrTy) { + // chosen to avoid collisions with existing globals in the module. Shared by + // the static-LDS `make_ptr` and dynamic-LDS `get_dyn_shared` lowerings. + static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, +- gpu::GPUModuleOp moduleOp, Location loc, StringRef prefix, ++ Block *moduleBody, Location loc, StringRef prefix, + int64_t nbytes, int64_t align, unsigned addrSpace) { + llvm::StringSet<> existingNames; +- for (auto globalOp : moduleOp.getBody()->getOps()) ++ for (auto globalOp : moduleBody->getOps()) + existingNames.insert(globalOp.getSymName()); + + unsigned counter = 0; +@@ -96,7 +97,7 @@ static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, + prefix, [&](StringRef candidate) { return existingNames.contains(candidate); }, counter); + + OpBuilder::InsertionGuard guard(rewriter); +- rewriter.setInsertionPointToStart(moduleOp.getBody()); ++ rewriter.setInsertionPointToStart(moduleBody); + + auto arrayTy = LLVM::LLVMArrayType::get(IntegerType::get(rewriter.getContext(), 8), nbytes); + auto globalOp = LLVM::GlobalOp::create(rewriter, loc, arrayTy, +@@ -149,12 +150,17 @@ public: + return rewriter.notifyMatchFailure( + op, "shared make_ptr requires allocBytes, allocAlign in dictAttrs"); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("shared make_ptr must be inside a gpu.module"); + + LLVM::GlobalOp global = +- createLDSGlobal(rewriter, moduleOp, loc, "__shared_alloc", allocBytesAttr.getInt(), ++ createLDSGlobal(rewriter, moduleBody, loc, "__shared_alloc", allocBytesAttr.getInt(), + allocAlignAttr.getInt(), /*addrSpace=*/3); + rewriter.replaceOpWithNewOp(op, global); + return success(); +@@ -191,15 +197,20 @@ public: + auto flyPtrTy = cast(op.getResult().getType()); + unsigned addrSpace = mapAttrToLLVMAddressSpace(flyPtrTy.getAddressSpace()); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("get_dyn_shared must be inside a gpu.module"); + + // Dynamic shared memory has a single logical region per kernel; reuse an + // existing `[0 x i8]` global if one is already present so multiple + // `get_dyn_shared` ops resolve to the same base symbol. + LLVM::GlobalOp sharedGlobal; +- for (auto globalOp : moduleOp.getBody()->getOps()) { ++ for (auto globalOp : moduleBody->getOps()) { + if (auto arrayType = dyn_cast(globalOp.getType())) { + if (globalOp.getAddrSpace() == addrSpace && arrayType.getNumElements() == 0 && + globalOp.getAlignment().value_or(0) == 1024) { +@@ -209,7 +220,7 @@ public: + } + } + if (!sharedGlobal) { +- sharedGlobal = createLDSGlobal(rewriter, moduleOp, loc, "__dynamic_shared", ++ sharedGlobal = createLDSGlobal(rewriter, moduleBody, loc, "__dynamic_shared", + /*nbytes=*/0, /*align=*/1024, addrSpace); + } + +@@ -810,10 +821,6 @@ public: + if (Value tok = op.getAsyncToken()) + asyncTokenType = tok.getType(); + +- // There are two relevant builder signatures in this MLIR: +- // - (kernel, ..., asyncTokenType, asyncDependencies, clusterSize) +- // - (kernel, ..., asyncObject, clusterSize) +- // Pick the one that matches the original op structure. + if (Value asyncObj = adaptor.getAsyncObject()) { + if (!adaptor.getAsyncDependencies().empty()) + return rewriter.notifyMatchFailure( +@@ -821,13 +828,14 @@ public: + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncObj, clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, ValueRange{}, asyncObj, clusterSize); + return success(); + } + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), ++ /*asyncObject=*/nullptr, clusterSize); + return success(); + } + }; +diff --git a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +--- a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp ++++ b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +@@ -2994,7 +2994,7 @@ + // A value reached twice on one walk (cyclic chain) cannot be bounded here. + if (!visited.insert(def).second) + return std::nullopt; +- auto pop = llvm::make_scope_exit([&] { visited.erase(def); }); ++ auto pop = llvm::scope_exit([&] { visited.erase(def); }); + + // Launch coordinates: ask the op for its own range. Every gpu index op + // implements InferIntRangeInterface, and that implementation already prefers diff --git a/third_party/flydsl/repo.bzl b/third_party/flydsl/repo.bzl new file mode 100644 index 00000000..144e5af5 --- /dev/null +++ b/third_party/flydsl/repo.bzl @@ -0,0 +1,17 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "flydsl", + remote = "https://github.com/ROCm/FlyDSL.git", + commit = "1941889400621f1fc3d8f03bada1a4e7380cdf7e", + build_file = "//third_party/flydsl:flydsl.bazel", + patches = [ + "//third_party/flydsl:atomic_exchange.patch", + "//third_party/flydsl:mlir_property_ref.patch", + "//third_party/flydsl:mlir_legacy_assembly.patch", + "//third_party/flydsl:vector_atomic.patch", + "//third_party/flydsl:xf32_mfma.patch", + ], + patch_args = ["-p1"], + ) diff --git a/third_party/flydsl/vector_atomic.patch b/third_party/flydsl/vector_atomic.patch new file mode 100644 index 00000000..edd23c40 --- /dev/null +++ b/third_party/flydsl/vector_atomic.patch @@ -0,0 +1,44 @@ +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -35,28 +35,36 @@ + return FxLayout(FxShape(FxC(1), FxC(getBitSize())), FxStride(FxC(1), FxC(1))); + } + ++static int32_t getAtomicValueBitWidth(Type type) { ++ if (auto vectorType = dyn_cast(type)) { ++ return vectorType.getNumElements() * ++ vectorType.getElementType().getIntOrFloatBitWidth(); ++ } ++ return type.getIntOrFloatBitWidth(); ++} ++ + bool CopyOpUniversalAtomicType::isStatic() const { return true; } + + Value CopyOpUniversalAtomicType::rebuildStaticValue(OpBuilder &builder, Location loc, + Value currentValue) const { + if (currentValue && isa(currentValue.getDefiningOp())) + return nullptr; +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return MakeCopyAtomOp::create(builder, loc, CopyAtomType::get(*this, bits), bits); + } + + Attribute CopyOpUniversalAtomicType::getThrLayout() const { return FxLayout(FxC(1), FxC(1)); } + + Attribute CopyOpUniversalAtomicType::getThrBitLayoutSrc() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutDst() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutRef() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + diff --git a/third_party/flydsl/xf32_mfma.patch b/third_party/flydsl/xf32_mfma.patch new file mode 100644 index 00000000..bc6126a0 --- /dev/null +++ b/third_party/flydsl/xf32_mfma.patch @@ -0,0 +1,25 @@ +diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +@@ -109,8 +109,12 @@ + static bool isBF8(Type ty) { return isa(ty); } + + static Type getMfmaABType(MLIRContext *ctx, Type elemTy, int32_t mn, int32_t k = 0) { +- if (elemTy.isF32()) ++ if (elemTy.isF32()) { ++ // v_mfma_f32_32x32x4_xf32 consumes two packed FP32 inputs per lane. ++ if (mn == 32 && k == 4) ++ return VectorType::get({2}, Float32Type::get(ctx)); + return Float32Type::get(ctx); ++ } + if (elemTy.isF16()) + return VectorType::get({mn * k / 64}, Float16Type::get(ctx)); + if (elemTy.isBF16()) { +@@ -179,6 +183,7 @@ + DISPATCH_MFMA_SSA(4, 1, elemTyA.isF32(), mfma_f32_4x4x1f32) + DISPATCH_MFMA_SSA(32, 2, elemTyA.isF32(), mfma_f32_32x32x2f32) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF32(), mfma_f32_16x16x4f32) ++ DISPATCH_MFMA_SSA(32, 4, elemTyA.isF32(), mfma_f32_32x32x4_xf32) + + DISPATCH_MFMA_SSA(32, 4, elemTyA.isF16(), mfma_f32_32x32x4f16) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF16(), mfma_f32_16x16x4f16) diff --git a/third_party/grpc/grpc++_unsecure-visibility.patch b/third_party/grpc/grpc++_unsecure-visibility.patch new file mode 100644 index 00000000..3b4013ab --- /dev/null +++ b/third_party/grpc/grpc++_unsecure-visibility.patch @@ -0,0 +1,13 @@ +diff --git a/BUILD b/BUILD +index 0e4bda7e16..dbe1708421 100644 +--- a/BUILD ++++ b/BUILD +@@ -1236,7 +1236,7 @@ grpc_cc_library( + "avoid_dep", + "nofixdeps", + ], +- visibility = ["//bazel:grpc++_unsecure"], ++ visibility = ["//visibility:public"], + deps = [ + "channel_arg_names", + "global_callback_hook", diff --git a/third_party/grpc/plugin-flags.patch b/third_party/grpc/plugin-flags.patch new file mode 100644 index 00000000..b69a77f5 --- /dev/null +++ b/third_party/grpc/plugin-flags.patch @@ -0,0 +1,19 @@ +# Match XLA's cc_grpc_library extension for services_namespace. +--- a/bazel/cc_grpc_library.bzl ++++ b/bazel/cc_grpc_library.bzl +@@ -29,6 +29,7 @@ + allow_deprecated = False, + use_external = False, # @unused + grpc_only = False, ++ plugin_flags = [], + **kwargs): + """Generates C++ grpc classes for services defined in a proto file. + +@@ -110,6 +111,7 @@ + well_known_protos = well_known_protos, + generate_mocks = generate_mocks, + allow_deprecated = allow_deprecated, ++ flags = plugin_flags, + **kwargs + ) + diff --git a/third_party/grpc/protobuf-35-upb-load.patch b/third_party/grpc/protobuf-35-upb-load.patch new file mode 100644 index 00000000..7311b981 --- /dev/null +++ b/third_party/grpc/protobuf-35-upb-load.patch @@ -0,0 +1,14 @@ +diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl +index e0d47a29e1..e33f40d758 100644 +--- a/bazel/grpc_build_system.bzl ++++ b/bazel/grpc_build_system.bzl +@@ -31,7 +31,8 @@ load("@build_bazel_apple_support//rules:universal_binary.bzl", "universal_binary + load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") + load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner") + load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +-load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library") ++load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", upb_proto_library = "upb_c_proto_library") ++load("@com_google_protobuf//bazel:upb_proto_reflection_library.bzl", "upb_proto_reflection_library") + load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + load("@rules_cc//cc:cc_test.bzl", "cc_test") diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel b/third_party/iree/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel rename to third_party/iree/BUILD.bazel diff --git a/third_party/iree/fix-added-token-matching.patch b/third_party/iree/fix-added-token-matching.patch new file mode 100644 index 00000000..2a418780 --- /dev/null +++ b/third_party/iree/fix-added-token-matching.patch @@ -0,0 +1,123 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +index 47d2a04c4d..66f1474c81 100644 +--- a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +@@ -48,11 +48,12 @@ static const iree_string_view_t kTopLevelAllowedKeys[] = { + + // Builds special_tokens collections from added_tokens. + // +-// ALL added tokens with `normalized=true` are included (not just those with +-// `special=true`) because they must be matched BEFORE the segmenter runs. +-// Without this, ByteLevel segmentation transforms the input before these +-// tokens can match, breaking tokenizers like GPT-NeoX that have multi-space +-// tokens (e.g., " " at ID 50276). ++// ALL added tokens are included (regardless of their `special` flag) because ++// they must be matched BEFORE the segmenter runs, mirroring HuggingFace's ++// AddedVocabulary. Without this, ByteLevel segmentation transforms the input ++// before these tokens can match, breaking tokenizers like GPT-NeoX that have ++// multi-space tokens (e.g., " " at ID 50276) and dropping non-special added ++// tokens such as `` or ``. + // + // Tokens are split by their normalized flag: + // - out_special_tokens: normalized=false, matched in raw input before +@@ -70,22 +71,10 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + iree_tokenizer_special_tokens_initialize(out_special_tokens); + iree_tokenizer_special_tokens_initialize(out_special_tokens_post_norm); + +- // Count tokens to process: special tokens OR tokens with normalized=true. +- // Non-special tokens with normalized=true must be matched before +- // segmentation. +- iree_host_size_t match_count = 0; +- for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { +- const iree_tokenizer_huggingface_added_token_t* token = +- iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); +- bool is_normalized = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); +- // Include if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- ++match_count; +- } +- } ++ // All added tokens are matched before segmentation, mirroring HuggingFace's ++ // AddedVocabulary which matches every added token regardless of its `special` ++ // flag. They are routed by their `normalized` flag below. ++ iree_host_size_t match_count = added_tokens->count; + + // Early exit if nothing to process. + if (match_count == 0) { +@@ -108,45 +97,38 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + i < added_tokens->count && iree_status_is_ok(status); ++i) { + const iree_tokenizer_huggingface_added_token_t* token = + iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); + bool is_normalized = iree_any_bit_set( + token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); + +- // Process if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- iree_string_view_t content = +- iree_tokenizer_huggingface_added_token_content(added_tokens, token); +- +- // Convert HuggingFace flags to special_tokens flags. +- iree_tokenizer_special_token_flags_t special_flags = +- IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; +- } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); + +- // Route to appropriate builder based on normalized flag. +- if (is_normalized) { +- // normalized=true: match after normalization but before segmentation. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_post_norm, content, token->id, special_flags); +- } else { +- // normalized=false (default): match before normalization. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_pre_norm, content, token->id, special_flags); +- } ++ // Convert HuggingFace flags to special_tokens flags. ++ iree_tokenizer_special_token_flags_t special_flags = ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; ++ } ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; ++ } ++ if (iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; ++ } ++ ++ // Route to appropriate builder based on normalized flag. ++ if (is_normalized) { ++ // normalized=true: match after normalization but before segmentation. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_post_norm, content, token->id, special_flags); ++ } else { ++ // normalized=false (default): match before normalization. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_pre_norm, content, token->id, special_flags); + } + } + diff --git a/third_party/iree/fix-gemma4-encode.patch b/third_party/iree/fix-gemma4-encode.patch new file mode 100644 index 00000000..18f493e6 --- /dev/null +++ b/third_party/iree/fix-gemma4-encode.patch @@ -0,0 +1,149 @@ +From 96151c5c077dc56c00e2b608c4f39b5553089515 Mon Sep 17 00:00:00 2001 +From: Benjamin Rabier +Date: Thu, 16 Apr 2026 09:07:02 +0000 +Subject: [PATCH] Fix tokenization + +Signed-off-by: Benjamin Rabier +--- + runtime/src/iree/tokenizer/tokenizer.c | 13 ++- + .../iree/tokenizer/tokenizer_encode_test.cc | 100 ++++++++++++++++++ + 2 files changed, 109 insertions(+), 4 deletions(-) + +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index ff9fe6242e..37df477c9d 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -2606,10 +2606,15 @@ static iree_status_t iree_tokenizer_encode_state_pump( + state->segmenter_view_start = state->read_position; + } + } +- // When a forced-complete drains the ring, exit partial mode so fresh +- // text after the special token goes through the normal segmenter path. +- if (force_complete && segment_complete && +- state->read_position >= state->write_position) { ++ // When a forced-complete segment finishes, retire the entire ring region ++ // that was just fed directly to the model. Unlike the partial reclaim path ++ // above, a non-partial completion resets the model to SEGMENT_START, so ++ // state_reclaim() returns 0 even though all bytes were fully processed. ++ // If we leave the ring positions unchanged, the same bytes are re-encoded ++ // on the next pump iteration and a pending special token can never flush. ++ if (force_complete && segment_complete) { ++ state->read_position = state->write_position; ++ state->segmenter_view_start = state->write_position; + state->has_partial_segment = false; + } + if (reclaimed > 0 || tokens_written > 0) { +diff --git a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +index a28811fe9b..0eb3f68dc8 100644 +--- a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc ++++ b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +@@ -3152,6 +3152,106 @@ TEST_F(TokenizerSpecialTokenStreamingTest, FinalizeDoesNotSilentlyDropPartial) { + iree_tokenizer_free(tokenizer); + } + ++// Regression test for a deferred pre-norm special token that causes an infinite ++// loop when the pump enters partial-segment mode. ++// ++// The bug: after a force-complete of ring content (triggered by a pending ++// special token), state_reclaim() returns 0 because the model resets to ++// SEGMENT_START on non-partial completion. The old code required ++// read_position >= write_position to clear partial mode and advance positions, ++// but since reclaim never advanced read_position, this condition was always ++// false — creating an infinite loop re-encoding the same bytes. ++// ++// This test reproduces the real-world trigger: a Replace normalizer that ++// expands " " → "▁" (1 byte → 3 bytes). With a Split segmenter on space ++// (MergedWithPrevious), each word+space is a small segment, but the normalizer ++// expansion means N bytes of input produce up to 3N bytes of normalized output. ++// When enough text with spaces precedes a special token, the normalized output ++// overflows the ring buffer's logical capacity, forcing partial-segment mode. ++// The pending special token then triggers force_complete, exposing the bug. ++// ++// With a 128-byte ring (64-byte logical capacity), ~40 bytes of space-heavy ++// input normalizes to >64 bytes, reliably triggering the overflow. ++TEST_F(TokenizerSpecialTokenStreamingTest, ++ DeferredSpecialTokenDoesNotReplayForcedCompleteSegment) { ++ // ▁ (U+2581) = 0xE2 0x96 0x81 (3 bytes in UTF-8). ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ ++ ScopedVocabBuilder vocab_builder(257); ++ for (int i = 0; i < 256; ++i) { ++ char byte_token[2] = {static_cast(i), '\0'}; ++ vocab_builder.AddToken(i, byte_token); ++ } ++ // Add ▁ as a token so the BPE model can encode normalized spaces. ++ vocab_builder.AddToken(256, kMetaspace); ++ vocab_builder.AddToken(50256, "<|tool>"); ++ ScopedVocab vocab = vocab_builder.Build(); ++ ++ iree_tokenizer_special_tokens_builder_t st_builder; ++ iree_tokenizer_special_tokens_builder_initialize(iree_allocator_system(), ++ &st_builder); ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_add( ++ &st_builder, IREE_SV("<|tool>"), 50256, ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE)); ++ iree_tokenizer_special_tokens_t special_tokens; ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_build( ++ &st_builder, iree_allocator_system(), &special_tokens)); ++ iree_tokenizer_special_tokens_builder_deinitialize(&st_builder); ++ ++ // Replace normalizer: " " → "▁" (1→3 byte expansion, matching Gemma4). ++ iree_tokenizer_normalizer_t* replace_normalizer = nullptr; ++ IREE_ASSERT_OK(iree_tokenizer_normalizer_replace_allocate( ++ IREE_SV(" "), iree_make_string_view(kMetaspace, 3), ++ iree_allocator_system(), &replace_normalizer)); ++ ++ // Split on literal space with MergedWithPrevious (matching Gemma4 config). ++ iree_tokenizer_segmenter_t* segmenter = CreateSplitSegmenterWithBehavior( ++ " ", IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_PREVIOUS); ++ ASSERT_NE(segmenter, nullptr); ++ ++ ScopedBuilder builder; ++ iree_tokenizer_builder_set_normalizer(builder.get(), replace_normalizer); ++ iree_tokenizer_builder_set_segmenter(builder.get(), segmenter); ++ iree_tokenizer_builder_set_model(builder.get(), CreateBPEModel(vocab.get())); ++ iree_tokenizer_builder_set_special_tokens(builder.get(), &special_tokens); ++ iree_tokenizer_builder_set_vocab(builder.get(), vocab.release()); ++ ++ iree_tokenizer_t* tokenizer = BuildTokenizer(builder.get()); ++ ASSERT_NE(tokenizer, nullptr); ++ ++ // Normal text with many spaces. Each space expands from 1→3 bytes after ++ // normalization. With a 128-byte ring (64-byte logical capacity), even this ++ // short text overflows the ring due to normalizer expansion, forcing ++ // partial-segment mode. The special token then triggers force_complete. ++ std::string input = ++ "A knight of powder-horn and shot " ++ "Once fill'd his bag as I would not " ++ "Unless the feelings of my breast " ++ "By poverty were sorely press'd " ++ "With birds and squirrels for the spits " ++ "Of certain gormandizing cits " ++ "With merry heart the fellow went "; ++ input += "<|tool>d"; ++ ++ IREE_ASSERT_OK_AND_ASSIGN(auto large_tokens, ++ EncodeWithBufferSize(tokenizer, input, 1024)); ++ IREE_ASSERT_OK_AND_ASSIGN(auto small_tokens, ++ EncodeWithBufferSize(tokenizer, input, 256)); ++ ++ EXPECT_EQ(small_tokens, large_tokens) ++ << "Small-buffer streaming must match large-buffer streaming when a " ++ "deferred special token follows text with normalizer expansion"; ++ ++ // Verify the special token and trailing byte are present at the end. ++ ASSERT_GE(small_tokens.size(), 2u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 2], 50256u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 1], ++ static_cast('d')); ++ ++ iree_tokenizer_special_tokens_deinitialize(&special_tokens); ++ iree_tokenizer_free(tokenizer); ++} ++ + //===----------------------------------------------------------------------===// + // Strip Normalizer + Special Token Integration Tests + //===----------------------------------------------------------------------===// +-- +2.53.0 + diff --git a/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch new file mode 100644 index 00000000..194a7b87 --- /dev/null +++ b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch @@ -0,0 +1,74 @@ +diff --git a/runtime/src/iree/tokenizer/normalizer/sequence.c b/runtime/src/iree/tokenizer/normalizer/sequence.c +--- a/runtime/src/iree/tokenizer/normalizer/sequence.c ++++ b/runtime/src/iree/tokenizer/normalizer/sequence.c +@@ -188,6 +188,14 @@ + "%" PRIhsz " exceeds maximum %" PRIu32, + flattened_count, + IREE_TOKENIZER_NORMALIZER_SEQUENCE_MAX_DEPTH); ++ } ++ ++ // Allocate before transferring ownership, including flattening nested ++ // sequences. On failure the caller must retain all original children. ++ iree_tokenizer_normalizer_sequence_t* normalizer = NULL; ++ if (flattened_count != 1) { ++ IREE_RETURN_IF_ERROR(iree_allocator_malloc( ++ allocator, sizeof(*normalizer), (void**)&normalizer)); + } + + // Build flattened children array, transferring ownership as we go. +@@ -232,18 +240,6 @@ + state_size = iree_host_align(state_size, iree_max_align_t); + child_state_offsets[i] = state_size; + state_size += iree_tokenizer_normalizer_state_size(flattened_children[i]); +- } +- +- // Allocate the sequence normalizer. +- iree_tokenizer_normalizer_sequence_t* normalizer = NULL; +- iree_status_t status = iree_allocator_malloc(allocator, sizeof(*normalizer), +- (void**)&normalizer); +- if (!iree_status_is_ok(status)) { +- // Cleanup: we've taken ownership of the children, so free them on failure. +- for (iree_host_size_t i = 0; i < flattened_count; ++i) { +- iree_tokenizer_normalizer_free(flattened_children[i]); +- } +- return status; + } + + // Initialize base. +@@ -566,6 +562,15 @@ + if (tile_size > IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH) { + // Limit tile to what we can handle in scratch. + tile_size = IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH; ++ // Never split a UTF-8 codepoint between child process calls. ++ while (tile_size > 0 && ++ ((uint8_t)remaining_input.data[tile_size] & 0xC0) == 0x80) { ++ --tile_size; ++ } ++ } ++ iree_tokenizer_normalizer_flags_t tile_flags = flags; ++ if (tile_size < remaining_input.size) { ++ tile_flags &= ~IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END; + } + + // Process one tile through all children. +@@ -574,7 +579,7 @@ + iree_host_size_t tile_consumed = 0; + iree_host_size_t tile_written = 0; + IREE_RETURN_IF_ERROR(iree_tokenizer_normalizer_sequence_process_tile( +- state, tile_input, scratch, scratch_size, remaining_output, flags, ++ state, tile_input, scratch, scratch_size, remaining_output, tile_flags, + &tile_consumed, &tile_written)); + + if (tile_consumed == 0 && tile_written == 0) { +diff --git a/runtime/src/iree/tokenizer/normalizer/strip.c b/runtime/src/iree/tokenizer/normalizer/strip.c +--- a/runtime/src/iree/tokenizer/normalizer/strip.c ++++ b/runtime/src/iree/tokenizer/normalizer/strip.c +@@ -353,7 +353,7 @@ + } + + // Phase 3: Handle segment end. +- if (is_segment_end) { ++ if (is_segment_end && in_ptr == in_end) { + // Pending whitespace is trailing - discard it. + iree_tokenizer_normalizer_strip_clear_pending(state); + state->at_start = true; diff --git a/third_party/iree/futex-public-api-header.patch b/third_party/iree/futex-public-api-header.patch new file mode 100644 index 00000000..7623f422 --- /dev/null +++ b/third_party/iree/futex-public-api-header.patch @@ -0,0 +1,17 @@ +diff --git a/runtime/src/iree/base/threading/futex.h b/runtime/src/iree/base/threading/futex.h +index 9b42d3cfd0..f2509b77e9 100644 +--- a/runtime/src/iree/base/threading/futex.h ++++ b/runtime/src/iree/base/threading/futex.h +@@ -30,11 +30,8 @@ + + #include + +-#include "iree/base/attributes.h" +-#include "iree/base/config.h" +-#include "iree/base/status.h" ++#include "iree/base/api.h" + #include "iree/base/target_platform.h" +-#include "iree/base/time.h" + + //===----------------------------------------------------------------------===// + // Platform detection diff --git a/third_party/iree/laguna-tokenizer-compat.patch b/third_party/iree/laguna-tokenizer-compat.patch new file mode 100644 index 00000000..c978c591 --- /dev/null +++ b/third_party/iree/laguna-tokenizer-compat.patch @@ -0,0 +1,78 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -217,0 +218,10 @@ ++ // Some code-oriented tokenizers use this lookahead split only to attach ++ // single-newline runs to the next segment. The IREE regex compiler does not ++ // support lookahead; the following ByteLevel/GPT split still preserves the ++ // input bytes, so treat this grouping hint as a no-op instead of rejecting ++ // the tokenizer at load time. ++ if (iree_string_view_equal( ++ pattern, IREE_SV("(?:\\r?\\n)+(?!\\r?\\n)")) && ++ behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_NEXT && !invert) { ++ return iree_ok_status(); ++ } +diff --git a/runtime/src/iree/tokenizer/format/huggingface/model_json.c b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/model_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +@@ -444,0 +445,46 @@ ++static char iree_tokenizer_ascii_upper(char c) { ++ return c >= 'a' && c <= 'z' ? (char)(c - ('a' - 'A')) : c; ++} ++ ++static bool iree_tokenizer_string_view_contains_ascii_case( ++ iree_string_view_t value, iree_string_view_t needle) { ++ if (needle.size == 0 || needle.size > value.size) return false; ++ for (iree_host_size_t i = 0; i <= value.size - needle.size; ++i) { ++ bool match = true; ++ for (iree_host_size_t j = 0; j < needle.size; ++j) { ++ if (iree_tokenizer_ascii_upper(value.data[i + j]) != ++ iree_tokenizer_ascii_upper(needle.data[j])) { ++ match = false; ++ break; ++ } ++ } ++ if (match) return true; ++ } ++ return false; ++} ++ ++static bool iree_tokenizer_huggingface_find_added_unk_token( ++ const iree_tokenizer_huggingface_added_tokens_t* added_tokens, ++ iree_tokenizer_token_id_t* out_token_id) { ++ *out_token_id = IREE_TOKENIZER_TOKEN_ID_INVALID; ++ if (!added_tokens) return false; ++ for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { ++ const iree_tokenizer_huggingface_added_token_t* token = ++ iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); ++ if (!iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL)) { ++ continue; ++ } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); ++ if (!iree_tokenizer_string_view_contains_ascii_case(content, ++ IREE_SV("unk"))) { ++ continue; ++ } ++ *out_token_id = token->id; ++ return true; ++ } ++ return false; ++} ++ +@@ -584,3 +630,10 @@ +- status = iree_make_status(IREE_STATUS_NOT_FOUND, +- "unk_token '%.*s' not found in vocabulary", +- (int)unk_length, unk_buffer); ++ // Some tokenizer.json files keep a stale default `unk_token` in the BPE ++ // model section while declaring the real unknown token in added_tokens ++ // (for example, custom bracketed forms such as `〈|UNK|〉`). Prefer that ++ // explicit special added token before failing the whole tokenizer load. ++ if (!iree_tokenizer_huggingface_find_added_unk_token(added_tokens, ++ &unk_token_id)) { ++ status = iree_make_status(IREE_STATUS_NOT_FOUND, ++ "unk_token '%.*s' not found in vocabulary", ++ (int)unk_length, unk_buffer); ++ } diff --git a/third_party/iree/match-hf-tokenizer.patch b/third_party/iree/match-hf-tokenizer.patch new file mode 100644 index 00000000..4cbab21a --- /dev/null +++ b/third_party/iree/match-hf-tokenizer.patch @@ -0,0 +1,1643 @@ +diff --git a/runtime/src/iree/tokenizer/model/bpe.c b/runtime/src/iree/tokenizer/model/bpe.c +index 309f1e261f..02fb055eee 100644 +--- a/runtime/src/iree/tokenizer/model/bpe.c ++++ b/runtime/src/iree/tokenizer/model/bpe.c +@@ -179,16 +179,17 @@ iree_status_t iree_tokenizer_bpe_model_allocate( + // Backtracking path capacities. + // Segments up to this size use the O(n) backtracking algorithm. + // Larger segments fall back to the O(n log L) window+heap path. +- // Capped at 4095 so the bitfield (ceil_div(4096, 64) = 64 words) fits in +- // a single uint64_t dirty mask for O(1) amortized per-segment init. ++ // Capped at 32768 to keep exact backtracking state bounded while covering ++ // large normalized prompts that still need HuggingFace-compatible ++ // suffix/prefix merge validation. + iree_host_size_t min_backtrack = 0; + if (!iree_host_size_checked_mul(16, model->max_token_length, + &min_backtrack)) { + // Overflow implies very large token length; clamp to maximum. +- min_backtrack = 4095; ++ min_backtrack = 32768; + } +- if (min_backtrack < 2048) min_backtrack = 2048; +- if (min_backtrack > 4095) min_backtrack = 4095; ++ if (min_backtrack < 32768) min_backtrack = 32768; ++ if (min_backtrack > 32768) min_backtrack = 32768; + model->max_backtrack_segment_bytes = min_backtrack; + // Stack holds at most one token per byte (worst case: all single-byte + // tokens). +@@ -311,6 +312,7 @@ static void iree_tokenizer_bpe_model_destroy( + iree_tokenizer_vocab_trie_free(model->trie); + iree_tokenizer_vocab_merge_hash_free(model->merge_hash); + iree_allocator_free(allocator, model->backtrack_tables.slab); ++ iree_allocator_free(allocator, model->backtrack_tables.alternate_suffix_slab); + iree_allocator_free(allocator, model); + + IREE_TRACE_ZONE_END(z0); +diff --git a/runtime/src/iree/tokenizer/model/bpe_backtrack.c b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +index 7f2cab797c..02b272838f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_backtrack.c ++++ b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +@@ -81,10 +81,9 @@ typedef struct iree_tokenizer_bpe_validation_frame_t { + static bool iree_tokenizer_bpe_is_valid_token_pair( + const iree_tokenizer_bpe_model_t* model, uint32_t token1, uint32_t token2, + uint32_t deferred_merge_rank) { +- const iree_tokenizer_vocab_t* vocab = model->vocab; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + + iree_tokenizer_bpe_validation_frame_t + stack[IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY]; +@@ -111,7 +110,7 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + (int32_t)frame->token1, + (int32_t)frame->token2); + if (merge.result_id >= 0) { +- uint32_t combined_rank = effective_rank[(uint32_t)merge.result_id]; ++ uint32_t combined_rank = merge.rank + 1; + // A merge is invalid if it should have fired (rank < limit) UNLESS + // it was intentionally deferred for a better suffix merge. When + // suffix_blocked triggers, it reports the rank of the blocked token; +@@ -140,7 +139,8 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + // Need to split at least one token. Determine order based on rank. + // Higher-ranked tokens (formed later in BPE) are split first. + bool try_token1_first = +- effective_rank[frame->token1] > effective_rank[frame->token2]; ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token1) > ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token2); + + if (try_token1_first) { + frame->side = +@@ -172,16 +172,26 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (split.left_id != decompose_token) { + // Non-base token: split_table provides (left_id, right_id) directly. + // By construction, merge(left_id, right_id) == decompose_token. ++ uint32_t decomposition_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, decompose_token); ++ iree_tokenizer_merge_hash_result_t split_merge = ++ iree_tokenizer_vocab_merge_hash_lookup( ++ model->merge_hash, (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (split_merge.result_id == (int32_t)decompose_token) { ++ decomposition_rank = split_merge.rank + 1; ++ } ++ + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = split.right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = split.left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + if (stack_count < IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY) { +@@ -243,16 +253,17 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (merge.result_id != (int32_t)decompose_token) continue; + + // Valid decomposition found. Compute new limit and push child frame. ++ uint32_t decomposition_rank = merge.rank + 1; + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = (uint32_t)right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = (uint32_t)left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + // Save resume position for backtracking. +@@ -381,160 +392,202 @@ static int32_t iree_tokenizer_bpe_find_first_token_at( + static int32_t iree_tokenizer_bpe_token_at_position( + const iree_tokenizer_bpe_model_t* model, const uint8_t* data, + iree_host_size_t size, iree_host_size_t* out_length) { +- int32_t token = iree_tokenizer_bpe_single_char_token(model, data[0]); +- if (token >= 0) { +- if (out_length) *out_length = 1; +- return token; ++ return iree_tokenizer_bpe_base_token_at_position(model, data, size, ++ out_length); ++} ++ ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank); ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank, int depth); ++ ++// Recursive implementation for iree_tokenizer_bpe_is_token_consumed_rightward. ++// ++// |ready_rank_plus_one| tracks when the right compound is available. Base ++// tokens are available at rank 0; a compound built by an internal merge at ++// rank N is available starting at N+1, so an outer merge at rank R can use it ++// iff ready_rank_plus_one <= R. ++static bool iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ iree_host_size_t token_end, uint32_t max_rank, int depth) { ++ if (token_end >= remaining_size) return false; ++ if (depth >= 8) return false; ++ ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ for (iree_host_size_t i = token_end; i < remaining_size; ++i) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, remaining_data[i], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t compound_token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (compound_token < 0) continue; ++ ++ iree_host_size_t compound_end = i + 1; ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, ++ compound_token); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge) || ++ merge.rank >= max_rank) { ++ continue; ++ } ++ ++ uint32_t compound = (uint32_t)compound_token; ++ bool compound_is_base = split_table[compound].left_id == compound; ++ uint32_t compound_ready_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, compound); ++ if (compound_ready_rank > merge.rank) continue; ++ if (!compound_is_base && ++ !iree_tokenizer_bpe_is_first_token_reachable(model, compound)) { ++ continue; ++ } ++ ++ if (!compound_is_base && compound_end < remaining_size) { ++ uint32_t rightmost_base = ++ iree_tokenizer_bpe_rightmost_base_token(model, compound); ++ uint32_t right_consumed = ++ iree_tokenizer_bpe_right_boundary_consumed_rank(model, compound); ++ if (right_consumed > 0 && ++ iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, (int32_t)rightmost_base, remaining_data, remaining_size, ++ compound_end, right_consumed - 1, depth + 1)) { ++ continue; ++ } ++ } ++ ++ uint32_t result_token = (uint32_t)merge.result_id; ++ if (compound_end < remaining_size && depth < 4) { ++ uint32_t blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, (int32_t)result_token, remaining_data + compound_end, ++ remaining_size - compound_end, NULL, 0, &blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } ++ ++ // merge(token, compound) would consume token if the compound is not itself ++ // consumed to the right before that merge rank. ++ if (!iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, compound_token, remaining_data, remaining_size, compound_end, ++ merge.rank, depth + 1)) { ++ return true; ++ } + } +- return iree_tokenizer_bpe_find_first_token_at(model, data, size, 8, +- out_length); ++ return false; + } + + // Checks whether a token at a given position is consumed by a rightward merge + // at rank lower than |max_rank|. This means merge(token, following) exists and + // fires before any merge at |max_rank| would. + // +-// To verify the rightward merge actually fires, also checks that the following +-// token isn't itself consumed by an even lower-rank merge. This handles chains +-// like: merge(c,h) would consume c, but merge(h,X) at lower rank consumes h +-// first — so merge(c,h) can't fire and c survives. ++// The following token may itself be a compound formed by lower-rank rightward ++// merges (for example t+ry after r+y has formed). This bounded search mirrors ++// the short compound lookahead used by suffix-blocking preemption. + static bool iree_tokenizer_bpe_is_token_consumed_rightward( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + iree_host_size_t token_end, uint32_t max_rank) { +- if (token_end >= remaining_size) return false; +- iree_host_size_t following_length = 1; +- int32_t following_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + token_end, remaining_size - token_end, +- &following_length); +- if (following_token < 0) return false; +- +- iree_tokenizer_merge_hash_result_t merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, +- following_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(merge) || +- merge.rank >= max_rank) { +- return false; +- } +- +- // merge(token, following) would fire. Verify following is available: if +- // merge(following, next_after) fires at even lower rank, following is +- // consumed and merge(token, following) can't fire. +- iree_host_size_t next_after_end = token_end + following_length; +- if (next_after_end < remaining_size) { +- int32_t next_after_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + next_after_end, remaining_size - next_after_end, +- NULL); +- if (next_after_token >= 0) { +- iree_tokenizer_merge_hash_result_t following_merge = +- iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, following_token, next_after_token); +- if (iree_tokenizer_merge_hash_result_is_valid(following_merge) && +- following_merge.rank < merge.rank) { +- return false; // Following consumed first, token survives. +- } +- } +- } +- return true; ++ return iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, token, remaining_data, remaining_size, token_end, max_rank, 0); + } + + // Checks if a suffix merge is preempted by lower-rank merges in the following + // input. Returns true if the suffix merge would NOT actually fire because the + // prefix token would be consumed first. + // +-// Two preemption paths: +-// Direct: merge(prefix, next) at rank < suffix_merge_rank, where the next +-// token is available (not consumed by a lower-rank rightward merge). +-// Compound: merge(prefix, compound) where compound is built by progressively +-// merging characters: next+third->c1, c1+fourth->c2, etc. This handles +-// cases where the direct merge can't fire but the prefix is still consumed +-// via a deeper compound (e.g., merge(l, ish) when i+s fires first). +-// +-// For non-byte-level tokenizers (e.g., SentencePiece), uses trie lookup to find +-// tokens when single-char lookup fails. This handles multi-byte base tokens +-// like metaspace (E2 96 81). ++// This delegates to the same bounded rightward-consumption search used to ++// validate prefix availability. That search handles direct and compound merges ++// while checking that each compound component is actually available at the rank ++// where it would be used. + static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + const iree_tokenizer_bpe_model_t* model, int32_t prefix_token, + uint32_t suffix_merge_rank, const uint8_t* remaining_data, + iree_host_size_t remaining_size, iree_host_size_t prefix_end) { +- if (prefix_end >= remaining_size) return false; +- +- iree_host_size_t next_length = 1; +- int32_t next_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + prefix_end, remaining_size - prefix_end, +- &next_length); +- if (next_token < 0) return false; +- +- // Direct preemption: merge(prefix, next) at rank < suffix_merge_rank. +- // Only valid if next is actually available (not consumed rightward). +- iree_tokenizer_merge_hash_result_t prefix_next = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- next_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_next) && +- prefix_next.rank < suffix_merge_rank) { +- if (!iree_tokenizer_bpe_is_token_consumed_rightward( +- model, next_token, remaining_data, remaining_size, +- prefix_end + next_length, prefix_next.rank)) { ++ return iree_tokenizer_bpe_is_token_consumed_rightward( ++ model, prefix_token, remaining_data, remaining_size, prefix_end, ++ suffix_merge_rank); ++} ++ ++static bool iree_tokenizer_bpe_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { + return true; + } + } ++ return false; ++} + +- // Compound preemption: build progressively longer compounds from the +- // characters following the prefix, checking merge(prefix, compound) at each +- // step. This catches cases where the direct merge can't fire but the prefix +- // merges with a multi-character compound that forms at lower rank. +- // +- // Example (Whisper "establish"): +- // prefix=l, next=i, merge(l,i) can't fire (i consumed by i+s at rank 15) +- // compound: i+s->is (rank 15), is+h->ish (rank 486) +- // merge(l, ish) at rank 1677 < suffix_merge_rank -> preempted +- iree_host_size_t compound_end = prefix_end + next_length; +- iree_host_size_t extension_length = 1; +- int32_t extension_token = -1; +- if (compound_end < remaining_size) { +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- } +- if (extension_token < 0) return false; ++static void iree_tokenizer_bpe_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; + +- iree_tokenizer_merge_hash_result_t compound_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, next_token, +- extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- return false; +- } ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; + +- // Build the compound progressively, checking at each level. +- int32_t compound_token = compound_merge.result_id; +- compound_end += extension_length; +- for (int depth = 0; depth < 4; ++depth) { +- iree_tokenizer_merge_hash_result_t prefix_compound = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- compound_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_compound) && +- prefix_compound.rank < suffix_merge_rank) { +- return true; ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; + } + +- // Extend compound by one more character. +- if (compound_end >= remaining_size) break; +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- if (extension_token < 0) break; +- +- compound_merge = iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, compound_token, extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- break; ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; + } +- compound_token = compound_merge.result_id; +- compound_end += extension_length; ++ ++ // Walk deeper suffixes of the right child. This generalizes the old ++ // rightmost split_table walk to alternate reachable decompositions, e.g. ++ // recon can split as rec+on or re+con and both suffixes can matter. ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); + } +- return false; + } + + // Checks if a candidate token is "suffix-blocked" - meaning a proper suffix of +@@ -566,21 +619,25 @@ static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + // This allows pair validation to accept pairs whose merge rank >= + // blocking_rank, since those merges were intentionally deferred for better + // suffix merges. +-static bool iree_tokenizer_bpe_is_suffix_blocked( ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + const char* suffix_data, iree_host_size_t suffix_length, +- uint32_t* out_blocking_rank) { ++ uint32_t* out_blocking_rank, int depth) { + *out_blocking_rank = 0; ++ if (depth >= 8) return false; + if (remaining_size == 0 && suffix_length == 0) return false; + + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ const uint64_t* token_has_reachable_alternates = ++ model->backtrack_tables.token_has_reachable_alternates; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + const bool byte_level = + iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); + +- uint32_t token_rank = effective_rank[(uint32_t)token]; ++ uint32_t token_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token); + if (token_rank <= 1) return false; // Base token, no suffix to check. + + // Collect all rightmost suffixes with their consumption ranks. +@@ -601,29 +658,64 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + uint32_t suffix_consumed_at[32]; + iree_host_size_t suffix_count = 0; + ++ bool needs_alternate_suffixes = false; + for (uint32_t current = (uint32_t)token; +- suffix_count < 32 && split_table[current].left_id != current; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; + current = split_table[current].right_id) { +- // Check if this decomposition is actually reachable via BPE. If the +- // split_table decomposition is blocked by a boundary merge, skip this +- // suffix - it won't exist in the actual BPE token sequence. ++ if (iree_any_bit_set(token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ needs_alternate_suffixes = true; ++ break; ++ } + uint32_t left = split_table[current].left_id; + uint32_t right = split_table[current].right_id; + bool reachable = iree_tokenizer_bpe_is_decomposition_reachable( + model, left, right, current); + if (!reachable) { +- // This decomposition is blocked. The actual BPE path uses a different +- // decomposition, so we can't rely on this suffix chain. Stop collecting +- // suffixes here - the actual suffixes depend on the alternate path which +- // we don't have easy access to. ++ needs_alternate_suffixes = true; + break; + } + suffixes[suffix_count] = right; +- suffix_consumed_at[suffix_count] = effective_rank[current]; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); + suffix_count++; ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, right, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ if (needs_alternate_suffixes) { ++ suffix_count = 0; ++ if (model->backtrack_tables.alternate_suffix_offsets) { ++ uint32_t token_id = (uint32_t)token; ++ uint32_t start = ++ model->backtrack_tables.alternate_suffix_offsets[token_id]; ++ uint32_t end = ++ model->backtrack_tables.alternate_suffix_offsets[token_id + 1]; ++ for (uint32_t i = start; ++ i < end && suffix_count < IREE_ARRAYSIZE(suffixes); ++i) { ++ suffixes[suffix_count] = ++ model->backtrack_tables.alternate_suffixes[i]; ++ suffix_consumed_at[suffix_count] = ++ model->backtrack_tables.alternate_suffix_consumed_at[i]; ++ ++suffix_count; ++ } ++ iree_host_size_t loaded_suffix_count = suffix_count; ++ for (iree_host_size_t i = 0; i < loaded_suffix_count; ++i) { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, suffixes[i], suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)token, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), ++ "suffix chain depth exceeds capacity"); + } +- IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), +- "suffix chain depth exceeds capacity"); ++ + if (suffix_count == 0) return false; + + // Walk the trie to enumerate all prefix tokens of the remaining input. +@@ -652,18 +744,33 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- // When the suffix token equals the prefix token, we have a repeating +- // pattern like consecutive metaspaces. In this case, any shorter token +- // would also face suffix blocking by the same pattern, creating an +- // infinite loop. The suffix blocking optimization doesn't help here +- // because there's no "better" tokenization to defer to. +- if ((int32_t)suffixes[s] == prefix_token) continue; ++ // If the prefix token would itself be split to enable a better suffix ++ // merge with following bytes, it is not available for this boundary ++ // merge. This handles chains such as ge|uffer|t where b+uffer would ++ // form "buffer" before ge+b, except "uffer" is blocked by er+t. ++ if (i + 1 < remaining_size && depth < 7 && ++ iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1) { ++ uint32_t prefix_blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, prefix_token, remaining_data + i + 1, ++ remaining_size - (i + 1), NULL, 0, &prefix_blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } + + // Check if the prefix would be consumed by lower-rank merges first. + if (iree_tokenizer_bpe_is_suffix_merge_preempted( +@@ -678,7 +785,8 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + // the rightmost base of the prefix merges with the following + // character at a rank lower than the prefix's internal consumption + // of that base, the prefix cannot form here. +- if (effective_rank[(uint32_t)prefix_token] > 1 && ++ if (iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1 && + i + 1 < remaining_size) { + uint32_t rightmost_base = iree_tokenizer_bpe_rightmost_base_token( + model, (uint32_t)prefix_token); +@@ -725,14 +833,18 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the token suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- if ((int32_t)suffixes[s] == prefix_token) continue; +- + // Suffixed tokens consume all remaining input + suffix bytes, so no + // preemption check is needed. The prefix_token represents the complete + // remaining segment (remaining_data + partial suffix) and nothing +@@ -745,6 +857,16 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + return false; + } + ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank) { ++ return iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, token, remaining_data, remaining_size, suffix_data, suffix_length, ++ out_blocking_rank, 0); ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: Core Algorithm + //===----------------------------------------------------------------------===// +@@ -774,18 +896,11 @@ void iree_tokenizer_bpe_backtrack_encode( + iree_tokenizer_bpe_state_pair_cache(state, model); + const uint32_t pair_cache_mask = (uint32_t)(model->pair_cache_capacity - 1); + +- // Reset only the bitfield words dirtied by the previous segment's +- // backtracking. On the first call, dirty_mask is pre-set to all-ones by +- // state_initialize, so this performs the equivalent of a full init. +- // On subsequent calls with no backtracking, dirty_mask is 0 and this +- // loop body executes zero times. +- uint64_t dirty = state->backtrack.dirty_mask; +- while (dirty) { +- iree_host_size_t word_index = +- (iree_host_size_t)iree_math_count_trailing_zeros_u64(dirty); +- bitfield[word_index] = UINT64_MAX; +- dirty &= dirty - 1; // Clear lowest set bit. +- } ++ // Reset the reachability bitfield for this segment. The exact backtracking ++ // path supports more than 64 bitfield words, so a single dirty mask is not ++ // expressive enough. ++ memset(bitfield, 0xFF, ++ model->backtrack_bitfield_capacity * sizeof(uint64_t)); + state->backtrack.dirty_mask = 0; + + iree_host_size_t stack_count = 0; +@@ -984,7 +1099,8 @@ void iree_tokenizer_bpe_backtrack_encode( + // fires first, blocking the path to "hello" as a single token. + bool token_unreachable = + token_raw_length > 1 && +- model->backtrack_tables.effective_rank[(uint32_t)token] > 1 && ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token) > ++ 1 && + !iree_any_bit_set( + model->backtrack_tables.token_reachable[(uint32_t)token / 64], + 1ull << ((uint32_t)token % 64)); +diff --git a/runtime/src/iree/tokenizer/model/bpe_encode.c b/runtime/src/iree/tokenizer/model/bpe_encode.c +index bf140ae97c..4fa090026f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_encode.c ++++ b/runtime/src/iree/tokenizer/model/bpe_encode.c +@@ -280,39 +280,29 @@ static iree_status_t iree_tokenizer_bpe_encode_segment( + byte_position < segment.size; ++byte_position) { + uint8_t input_byte = (uint8_t)segment.data[byte_position]; + +- // Look up single-byte token. For ByteLevel mode, this is the UTF-8 +- // encoding of the ByteLevel codepoint (e.g., space -> "Ġ"). +- int32_t token_id = model->byte_to_token[input_byte]; ++ // Seed with the shortest base vocabulary token. Some base symbols (for ++ // example SentencePiece's U+2581 metaspace) span multiple UTF-8 bytes, ++ // and byte-fallback vocabularies may also have tokens for each raw byte. ++ // The BPE heap must start from the base symbol, not the byte fallback. + iree_host_size_t token_byte_length = 1; ++ int32_t token_id = iree_tokenizer_bpe_base_token_at_position( ++ model, (const uint8_t*)segment.data + byte_position, ++ segment.size - byte_position, &token_byte_length); + if (token_id < 0) { +- // No direct single-byte token. Try a multi-byte trie match first: +- // some base vocabulary tokens (e.g., SentencePiece's ▁ = U+2581, +- // 3 bytes) span multiple raw bytes but are never produced by a merge. +- // The trie walk finds the longest such token; effective_rank filtering +- // inside backtrack_longest_match ensures we only accept tokens that +- // participate in BPE (not stray added tokens). +- iree_host_size_t trie_raw_length = 0; +- iree_tokenizer_bpe_backtrack_longest_match( +- model, (const uint8_t*)segment.data + byte_position, +- segment.size - byte_position, &token_id, &trie_raw_length); +- if (token_id >= 0) { +- token_byte_length = trie_raw_length; +- } else { +- // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. +- // For FUSE_UNK, check the last token in the window (not yet emitted) +- // before falling back to last_emitted_token_id. +- int32_t previous_token = +- (state->window.count > 0) +- ? iree_tokenizer_bpe_window_at(state, model, +- state->window.count - 1) +- ->token_id +- : state->last_emitted_token_id; +- token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, +- previous_token); +- if (token_id < 0) { +- // Byte was fused with previous UNK. Skip to next byte. +- continue; +- } ++ // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. For ++ // FUSE_UNK, check the last token in the window (not yet emitted) ++ // before falling back to last_emitted_token_id. ++ int32_t previous_token = ++ (state->window.count > 0) ++ ? iree_tokenizer_bpe_window_at(state, model, ++ state->window.count - 1) ++ ->token_id ++ : state->last_emitted_token_id; ++ token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, ++ previous_token); ++ if (token_id < 0) { ++ // Byte was fused with previous UNK. Skip to next byte. ++ continue; + } + } + +@@ -620,13 +610,14 @@ iree_host_size_t iree_tokenizer_bpe_state_reclaim( + window[index].end_byte -= (uint32_t)committed; + } + +- // Adjust heap entry byte positions. +- // After BYTE_LOOP processes all segment bytes, apply_pending_merges has +- // drained most entries. Any remaining entries reference current window tokens +- // (left_start_byte >= committed). Stale entries that would underflow are +- // discarded by lazy invalidation when popped. +- for (iree_host_size_t i = 0; i < bpe_state->heap.size; ++i) { +- bpe_state->heap.entries[i].left_start_byte -= (uint32_t)committed; ++ // Rebuild heap entries for the rebased window. The heap can contain stale ++ // entries for tokens that were already emitted; subtracting |committed| from ++ // those would underflow their start byte and leave noisy stale entries. The ++ // window is bounded, so rebuilding is cheap and keeps all held-back merges ++ // visible after reclaim. ++ iree_tokenizer_bpe_heap_reset(&bpe_state->heap); ++ for (iree_host_size_t i = 0; i + 1 < bpe_state->window.count; ++i) { ++ iree_tokenizer_bpe_maybe_add_merge(bpe_state, model, i); + } + + return committed; +diff --git a/runtime/src/iree/tokenizer/model/bpe_internal.h b/runtime/src/iree/tokenizer/model/bpe_internal.h +index 69546df3f0..4e9873e653 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_internal.h ++++ b/runtime/src/iree/tokenizer/model/bpe_internal.h +@@ -45,50 +45,42 @@ typedef struct iree_tokenizer_bpe_split_entry_t { + uint32_t right_id; + } iree_tokenizer_bpe_split_entry_t; + +-// Backtracking stack entry: token ID and the byte position where it starts. +-// Bit-packed backtrack entry to maintain 8-byte size while supporting large +-// vocabs. start_byte is bounded by max_backtrack_segment_bytes (<=4095), so +-// we use 12 bits for it and 20 bits for deferred_merge_rank (supports 1M +-// tokens, covering all realistic vocabularies). +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS 12 +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK \ +- ((1u << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS) - 1) +-#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 20 ++// Backtracking stack entry: token ID, byte start, and deferred merge rank. ++// Keep deferred_merge_rank as a full 19-bit value (Gemma-sized vocabularies ++// need it) and store start_byte separately so exact backtracking can cover ++// larger normalized segments without falling back to the approximate heap path. ++#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 19 + #define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX \ + ((1u << IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS) - 1) + + typedef struct iree_tokenizer_bpe_backtrack_entry_t { + int32_t token_id; +- // Packed field: lower 12 bits = start_byte, upper 20 bits = deferred_rank. ++ uint32_t start_byte; + // The deferred_merge_rank is the effective_rank that was active when this + // token was pushed. It tells pair validation at the next position which + // merges were intentionally deferred due to suffix blocking. +- uint32_t start_byte_and_deferred_rank; ++ uint32_t deferred_rank; + } iree_tokenizer_bpe_backtrack_entry_t; + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_start_byte( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank & +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK; ++ return entry->start_byte; + } + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_deferred_rank( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank >> +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS; ++ return entry->deferred_rank; + } + + static inline void iree_tokenizer_bpe_backtrack_entry_set( + iree_tokenizer_bpe_backtrack_entry_t* entry, int32_t token_id, + uint32_t start_byte, uint32_t deferred_rank) { + entry->token_id = token_id; +- // Cap deferred_rank at 20-bit max (supports vocabs up to 1M tokens). + if (deferred_rank > IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX) { + deferred_rank = IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX; + } +- entry->start_byte_and_deferred_rank = +- (start_byte & IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK) | +- (deferred_rank << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS); ++ entry->start_byte = start_byte; ++ entry->deferred_rank = deferred_rank; + } + + //===----------------------------------------------------------------------===// +@@ -101,11 +93,19 @@ static inline void iree_tokenizer_bpe_backtrack_entry_set( + typedef struct iree_tokenizer_bpe_backtrack_tables_t { + // Single allocation for all tables below. + void* slab; ++ void* alternate_suffix_slab; + iree_tokenizer_bpe_split_entry_t* split_table; // Inverse merge table. + // Longest proper prefix token, or UINT32_MAX. + uint32_t* next_prefix_match; + uint32_t* effective_rank; // 0 = non-participant, >0 = BPE-reachable. + uint64_t* token_reachable; // Precomputed reachability bits. ++ // Tokens with more than one reachable decomposition. Suffix blocking must ++ // inspect alternate decompositions for these tokens instead of only walking ++ // split_table's right spine. ++ uint64_t* token_has_reachable_alternates; ++ uint32_t* alternate_suffix_offsets; // vocab_capacity + 1 entries. ++ uint32_t* alternate_suffixes; ++ uint32_t* alternate_suffix_consumed_at; + } iree_tokenizer_bpe_backtrack_tables_t; + + //===----------------------------------------------------------------------===// +@@ -118,10 +118,9 @@ typedef struct iree_tokenizer_bpe_backtrack_state_t { + iree_host_size_t stack_count; // Tokens in the stack. + // Next token to emit from stack. + iree_host_size_t emit_index; +- // Dirty mask for lazy bitfield reset. Each bit corresponds to a uint64_t +- // word in the backtrack bitfield that was modified (cleared) during the +- // previous segment's backtracking. On segment start, only these words are +- // reset to UINT64_MAX, giving O(backtracks) init instead of O(capacity). ++ // Retained for state layout compatibility with older code paths. The ++ // backtracking bitfield is reset as a whole at segment start because the ++ // exact path now supports more than 64 bitfield words. + uint64_t dirty_mask; + } iree_tokenizer_bpe_backtrack_state_t; + +@@ -210,6 +209,22 @@ typedef struct iree_tokenizer_bpe_model_t { + iree_host_size_t pair_cache_offset; + } iree_tokenizer_bpe_model_t; + ++static inline uint32_t iree_tokenizer_bpe_token_formation_rank( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token) { ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token]; ++ if (split.left_id == token) return 0; // Base token. ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (merge.result_id == (int32_t)token) { ++ return merge.rank + 1; ++ } ++ return model->backtrack_tables.effective_rank[token]; ++} ++ + //===----------------------------------------------------------------------===// + // Segment Context (State-Side) + //===----------------------------------------------------------------------===// +@@ -589,9 +604,9 @@ static inline bool iree_tokenizer_bpe_bitfield_is_set( + + static inline void iree_tokenizer_bpe_bitfield_clear( + uint64_t* bitfield, uint64_t* dirty_mask, iree_host_size_t position) { ++ (void)dirty_mask; + iree_host_size_t word_index = position / 64; + bitfield[word_index] &= ~((uint64_t)1 << (position % 64)); +- *dirty_mask |= (uint64_t)1 << word_index; + } + + //===----------------------------------------------------------------------===// +@@ -679,6 +694,73 @@ iree_tokenizer_bpe_backtrack_longest_match( + } + } + ++// Finds the shortest base vocabulary token matching raw input bytes at data[0]. ++// BPE starts from base symbols and applies merge rules by rank. For ++// SentencePiece-style tokenizers, some base symbols (for example U+2581 ++// metaspace) span multiple UTF-8 bytes and cannot be found through byte_to_token. ++// The window/heap path must seed those base symbols, not the longest reachable ++// merged token, otherwise rank-priority merges can be skipped. ++static inline int32_t iree_tokenizer_bpe_base_token_at_position( ++ const iree_tokenizer_bpe_model_t* model, const uint8_t* data, ++ iree_host_size_t size, iree_host_size_t* out_raw_length) { ++ if (size == 0) return -1; ++ ++ int32_t byte_token = model->byte_to_token[data[0]]; ++ ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ int32_t fallback_base_token = -1; ++ iree_host_size_t fallback_base_length = 0; ++ for (iree_host_size_t raw_consumed = 0; raw_consumed < size; ++ ++raw_consumed) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, data[raw_consumed], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (token < 0) continue; ++ ++ const uint32_t token_id = (uint32_t)token; ++ if (token_id >= model->vocab_capacity) continue; ++ ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token_id]; ++ const bool is_base_token = ++ split.left_id == token_id && split.right_id == token_id; ++ const bool is_matchable = ++ model->backtrack_tables.effective_rank[token_id] > 0; ++ if (is_base_token && is_matchable) { ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(model->vocab, token); ++ if (raw_consumed == 0 && data[0] >= 0x80 && token_text.size == 1 && ++ raw_consumed + 1 < size) { ++ fallback_base_token = token; ++ fallback_base_length = raw_consumed + 1; ++ continue; ++ } ++ if (out_raw_length) *out_raw_length = raw_consumed + 1; ++ return token; ++ } ++ } ++ ++ if (fallback_base_token >= 0) { ++ if (out_raw_length) *out_raw_length = fallback_base_length; ++ return fallback_base_token; ++ } ++ ++ if (byte_token >= 0) { ++ if (out_raw_length) *out_raw_length = 1; ++ return byte_token; ++ } ++ ++ return -1; ++} ++ + //===----------------------------------------------------------------------===// + // Table Construction (bpe_tables.c) + //===----------------------------------------------------------------------===// +diff --git a/runtime/src/iree/tokenizer/model/bpe_tables.c b/runtime/src/iree/tokenizer/model/bpe_tables.c +index c44021d3e1..480d7cc4e8 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_tables.c ++++ b/runtime/src/iree/tokenizer/model/bpe_tables.c +@@ -13,6 +13,28 @@ + #include "iree/tokenizer/vocab/vocab_merge_hash.h" + #include "iree/tokenizer/vocab/vocab_trie.h" + ++static bool iree_tokenizer_bpe_is_repeated_initial_symbol( ++ iree_string_view_t text) { ++ if (text.size < 2) return false; ++ ++ bool all_newlines = true; ++ bool all_tabs = true; ++ for (iree_host_size_t i = 0; i < text.size; ++i) { ++ all_newlines &= text.data[i] == '\n'; ++ all_tabs &= text.data[i] == '\t'; ++ } ++ if (all_newlines || all_tabs) return true; ++ ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ if (text.size % (sizeof(kMetaspace) - 1) != 0) return false; ++ for (iree_host_size_t i = 0; i < text.size; i += sizeof(kMetaspace) - 1) { ++ if (memcmp(text.data + i, kMetaspace, sizeof(kMetaspace) - 1) != 0) { ++ return false; ++ } ++ } ++ return true; ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: First Token Reachability + //===----------------------------------------------------------------------===// +@@ -56,7 +78,6 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the right spine until we find where the rightmost base token + // is consumed (i.e., where the right child is a base token). +@@ -64,7 +85,7 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + uint32_t right = split_table[token].right_id; + if (split_table[right].left_id == right) { + // Right child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into right subtree. + token = right; +@@ -81,7 +102,6 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the left spine until we find where the leftmost base token + // is consumed (i.e., where the left child is a base token). +@@ -89,7 +109,7 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + uint32_t left = split_table[token].left_id; + if (split_table[left].left_id == left) { + // Left child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into left subtree. + token = left; +@@ -121,6 +141,67 @@ bool iree_tokenizer_bpe_is_first_token_reachable( + 1ull << (token % 64)); + } + ++static bool iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, ++ uint32_t token, uint32_t token_rank) { ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ for (uint32_t current = left; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].right_id) { ++ suffixes[suffix_count] = split_table[current].right_id; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ suffix_count++; ++ } ++ if (suffix_count == 0) return false; ++ ++ uint32_t prefixes[32]; ++ uint32_t prefix_consumed_at[32]; ++ iree_host_size_t prefix_count = 0; ++ prefixes[prefix_count] = right; ++ prefix_consumed_at[prefix_count] = UINT32_MAX; ++ ++prefix_count; ++ for (uint32_t current = right; ++ prefix_count < IREE_ARRAYSIZE(prefixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].left_id) { ++ prefixes[prefix_count] = split_table[current].left_id; ++ prefix_consumed_at[prefix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++prefix_count; ++ } ++ ++ for (iree_host_size_t p = 0; p < prefix_count; ++p) { ++ for (iree_host_size_t s = 0; s < suffix_count; ++s) { ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)suffixes[s], ++ (int32_t)prefixes[p]); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge)) continue; ++ if ((uint32_t)merge.result_id == token) continue; ++ ++ uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, prefixes[p]); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; ++ if (merge_effective_rank >= token_rank) continue; ++ if (merge_effective_rank >= suffix_consumed_at[s]) continue; ++ if (merge_effective_rank > prefix_consumed_at[p]) continue; ++ return true; ++ } ++ } ++ return false; ++} ++ + // Checks if a specific decomposition (left + right) of a token is reachable. + // This is separate from is_first_token_reachable to allow checking alternative + // decompositions when a token has multiple merges producing it. +@@ -128,6 +209,22 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, + uint32_t token) { + const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ uint32_t decomposition_rank = effective_rank[token]; ++ iree_tokenizer_merge_hash_result_t decomposition_merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, (int32_t)left, ++ (int32_t)right); ++ if (decomposition_merge.result_id == (int32_t)token) { ++ decomposition_rank = decomposition_merge.rank + 1; ++ } ++ ++ uint32_t left_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, left); ++ uint32_t right_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, right); ++ if (decomposition_rank < left_available_at || ++ decomposition_rank < right_available_at) { ++ return false; ++ } + + // Check for blocking merge at the split boundary. + // +@@ -158,17 +255,18 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + if (boundary_merge.result_id >= 0 && + (uint32_t)boundary_merge.result_id != token) { + // There's a different merge at this boundary. It blocks only if it fires +- // before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = effective_rank[(uint32_t)boundary_merge.result_id]; ++ // before the left boundary token is consumed by an internal merge and ++ // before or tied with the right boundary token's internal merge. Ties are ++ // resolved left-to-right: an internal merge in the left token starts before ++ // this boundary and wins the tie, while an internal merge in the right ++ // token starts after this boundary and loses the tie. ++ uint32_t boundary_rank = boundary_merge.rank + 1; + uint32_t left_boundary_consumed = + iree_tokenizer_bpe_right_boundary_consumed_rank(model, left); + uint32_t right_boundary_consumed = + iree_tokenizer_bpe_left_boundary_consumed_rank(model, right); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { ++ if (boundary_rank < left_boundary_consumed && ++ boundary_rank <= right_boundary_consumed) { + return false; // Blocking merge fires before either boundary is consumed. + } + } +@@ -180,7 +278,74 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + iree_tokenizer_bpe_is_first_token_reachable(model, left); + bool right_reachable = + iree_tokenizer_bpe_is_first_token_reachable(model, right); +- return left_reachable && right_reachable; ++ if (!left_reachable || !right_reachable) return false; ++ ++ return !iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ model, left, right, token, decomposition_rank); ++} ++ ++static bool iree_tokenizer_bpe_table_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static void iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; ++ ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; ++ } ++ ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_table_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; ++ } ++ ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); ++ } + } + + //===----------------------------------------------------------------------===// +@@ -213,6 +378,7 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + iree_host_size_t next_prefix_offset = 0; + iree_host_size_t effective_rank_offset = 0; + iree_host_size_t token_reachable_offset = 0; ++ iree_host_size_t token_has_reachable_alternates_offset = 0; + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, + IREE_STRUCT_LAYOUT( +@@ -222,7 +388,9 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &next_prefix_offset), + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &effective_rank_offset), + IREE_STRUCT_FIELD(reachable_words, uint64_t, +- &token_reachable_offset))); ++ &token_reachable_offset), ++ IREE_STRUCT_FIELD(reachable_words, uint64_t, ++ &token_has_reachable_alternates_offset))); + + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_allocator_malloc(model->allocator, slab_size, +@@ -237,6 +405,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + (uint32_t*)(slab + effective_rank_offset); + model->backtrack_tables.token_reachable = + (uint64_t*)(slab + token_reachable_offset); ++ model->backtrack_tables.token_has_reachable_alternates = ++ (uint64_t*)(slab + token_has_reachable_alternates_offset); + + // Initialize: base tokens map to (self, self), rank 0, no prefix. + for (iree_host_size_t i = 0; i < vocab_capacity; ++i) { +@@ -309,6 +479,30 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Gemma-style BPE vocabularies contain repeated initial-symbol tokens ++ // (metaspace, newline, and tab runs) whose merge entries are not a ++ // rank-ordered DAG: longer runs can appear before the shorter runs needed to ++ // build them. HuggingFace still matches these as direct vocabulary symbols. ++ // Treat them as base tokens so reachability and suffix blocking do not try ++ // to validate impossible synthetic merge trees. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ iree_tokenizer_token_attr_t attrs = ++ iree_tokenizer_vocab_token_attrs(vocab, (int32_t)token_id); ++ if (attrs & IREE_TOKENIZER_TOKEN_ATTR_SPECIAL) continue; ++ ++ iree_string_view_t text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token_id); ++ if (!iree_tokenizer_bpe_is_repeated_initial_symbol(text)) continue; ++ ++ model->backtrack_tables.split_table[token_id].left_id = ++ (uint32_t)token_id; ++ model->backtrack_tables.split_table[token_id].right_id = ++ (uint32_t)token_id; ++ if (model->backtrack_tables.effective_rank[token_id] == 0) { ++ model->backtrack_tables.effective_rank[token_id] = 1; ++ } ++ } ++ + // Mark multi-byte merge participants as matchable. + // + // Some base vocabulary tokens (e.g., SentencePiece's ▁ character, U+2581) +@@ -433,6 +627,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + // and no blocking boundary merge. + memset(model->backtrack_tables.token_reachable, 0, + reachable_words * sizeof(uint64_t)); ++ memset(model->backtrack_tables.token_has_reachable_alternates, 0, ++ reachable_words * sizeof(uint64_t)); + + // Initialize: mark single-codepoint base tokens as reachable. + for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { +@@ -448,7 +644,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + codepoint_count++; + } + } +- if (codepoint_count <= 1) { ++ if (codepoint_count <= 1 || ++ iree_tokenizer_bpe_is_repeated_initial_symbol(token_text)) { + model->backtrack_tables.token_reachable[token_id / 64] |= + 1ull << (token_id % 64); + } +@@ -491,36 +688,10 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + continue; + } + +- // Check for blocking merge at the boundary. +- uint32_t left_boundary = +- iree_tokenizer_bpe_rightmost_base_token(model, merge.left_id); +- uint32_t right_boundary = +- iree_tokenizer_bpe_leftmost_base_token(model, merge.right_id); +- +- iree_tokenizer_merge_hash_result_t boundary_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, +- (int32_t)left_boundary, +- (int32_t)right_boundary); +- if (boundary_merge.result_id >= 0 && +- (uint32_t)boundary_merge.result_id != (uint32_t)result_id) { +- // There's a different merge at this boundary. It blocks only if it +- // fires before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = +- model->backtrack_tables +- .effective_rank[(uint32_t)boundary_merge.result_id]; +- uint32_t left_boundary_consumed = +- iree_tokenizer_bpe_right_boundary_consumed_rank(model, +- merge.left_id); +- uint32_t right_boundary_consumed = +- iree_tokenizer_bpe_left_boundary_consumed_rank(model, +- merge.right_id); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { +- continue; // Blocking merge fires first. +- } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ (uint32_t)result_id)) { ++ continue; + } + + // This decomposition is reachable! +@@ -549,6 +720,175 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Mark tokens that have reachable decompositions besides the canonical ++ // split_table decomposition. Most tokens have exactly one reachable path and ++ // can use the cheap right-spine suffix walk during encode; ambiguous tokens ++ // need the slower all-splits suffix collection for HuggingFace parity. ++ for (iree_host_size_t rank = 0; rank < merge_count; ++rank) { ++ iree_tokenizer_merge_t merge = iree_tokenizer_vocab_merge(vocab, rank); ++ iree_tokenizer_merge_hash_result_t merge_result = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ merge.left_id, merge.right_id); ++ int32_t result_id = merge_result.result_id; ++ if (result_id < 0 || (iree_host_size_t)result_id >= vocab_capacity) { ++ continue; ++ } ++ ++ uint32_t result = (uint32_t)result_id; ++ if (!iree_any_bit_set(model->backtrack_tables.token_reachable[result / 64], ++ 1ull << (result % 64))) { ++ continue; ++ } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ result)) { ++ continue; ++ } ++ ++ iree_tokenizer_bpe_split_entry_t canonical = ++ model->backtrack_tables.split_table[result]; ++ if (canonical.left_id != (uint32_t)merge.left_id || ++ canonical.right_id != (uint32_t)merge.right_id) { ++ model->backtrack_tables.token_has_reachable_alternates[result / 64] |= ++ 1ull << (result % 64); ++ } ++ } ++ ++ // Propagate the ambiguity bit to any token whose right-spine suffixes pass ++ // through an ambiguous token. Those parent tokens also need the precomputed ++ // all-splits suffix list during suffix blocking. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ for (uint32_t current = (uint32_t)token_id; ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ if (iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ break; ++ } ++ } ++ } ++ ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ uint32_t cheap_suffixes[32]; ++ uint32_t cheap_consumed_at[32]; ++ iree_host_size_t cheap_count = 0; ++ for (uint32_t current = (uint32_t)token_id; ++ cheap_count < IREE_ARRAYSIZE(cheap_suffixes) && ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ cheap_suffixes[cheap_count] = ++ model->backtrack_tables.split_table[current].right_id; ++ cheap_consumed_at[cheap_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++cheap_count; ++ } ++ ++ uint32_t all_suffixes[32]; ++ uint32_t all_consumed_at[32]; ++ iree_host_size_t all_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, all_suffixes, all_consumed_at, &all_count, ++ IREE_ARRAYSIZE(all_suffixes), 0); ++ ++ bool differs = cheap_count != all_count; ++ for (iree_host_size_t i = 0; i < all_count && !differs; ++i) { ++ bool found = false; ++ for (iree_host_size_t j = 0; j < cheap_count; ++j) { ++ if (all_suffixes[i] == cheap_suffixes[j] && ++ all_consumed_at[i] == cheap_consumed_at[j]) { ++ found = true; ++ break; ++ } ++ } ++ differs = !found; ++ } ++ if (differs) { ++ model->backtrack_tables.token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ } ++ } ++ ++ iree_host_size_t total_alternate_suffixes = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ total_alternate_suffixes += suffix_count; ++ } ++ ++ if (total_alternate_suffixes > 0) { ++ iree_host_size_t alt_slab_size = 0; ++ iree_host_size_t alt_offsets_offset = 0; ++ iree_host_size_t alt_suffixes_offset = 0; ++ iree_host_size_t alt_consumed_offset = 0; ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, ++ IREE_STRUCT_LAYOUT( ++ /*base_size=*/0, &alt_slab_size, ++ IREE_STRUCT_FIELD(vocab_capacity + 1, uint32_t, ++ &alt_offsets_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_suffixes_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_consumed_offset))); ++ ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, iree_allocator_malloc(model->allocator, alt_slab_size, ++ &model->backtrack_tables ++ .alternate_suffix_slab)); ++ ++ uint8_t* alt_slab = ++ (uint8_t*)model->backtrack_tables.alternate_suffix_slab; ++ model->backtrack_tables.alternate_suffix_offsets = ++ (uint32_t*)(alt_slab + alt_offsets_offset); ++ model->backtrack_tables.alternate_suffixes = ++ (uint32_t*)(alt_slab + alt_suffixes_offset); ++ model->backtrack_tables.alternate_suffix_consumed_at = ++ (uint32_t*)(alt_slab + alt_consumed_offset); ++ ++ iree_host_size_t write_index = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++ ++token_id) { ++ model->backtrack_tables.alternate_suffix_offsets[token_id] = ++ (uint32_t)write_index; ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, ++ &suffix_count, IREE_ARRAYSIZE(suffixes), 0); ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ model->backtrack_tables.alternate_suffixes[write_index] = suffixes[i]; ++ model->backtrack_tables.alternate_suffix_consumed_at[write_index] = ++ suffix_consumed_at[i]; ++ ++write_index; ++ } ++ } ++ model->backtrack_tables.alternate_suffix_offsets[vocab_capacity] = ++ (uint32_t)write_index; ++ } ++ + IREE_TRACE_ZONE_END(z0); + return iree_ok_status(); + } +diff --git a/runtime/src/iree/tokenizer/model/bpe_test.cc b/runtime/src/iree/tokenizer/model/bpe_test.cc +index f2b82260d5..5bac3f0f44 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_test.cc ++++ b/runtime/src/iree/tokenizer/model/bpe_test.cc +@@ -82,13 +82,13 @@ TEST_F(BPEModelTest, StateSizeIsReasonable) { + // tokens. For max_token_length=1, vocab_capacity=1: + // window: 2 * 12 = 24 + // heap: 3 * 8 = 24 +- // backtrack stack: 2048 * 8 = 16,384 +- // backtrack bitfield: 33 * 8 = 264 ++ // backtrack stack: 32768 * 12 = 393,216 ++ // backtrack bitfield: 513 * 8 = 4,104 + // pair cache: 4096 * 8 = 32,768 + // word cache: 0 (disabled, vocab < 256) + // struct: ~128 +- // total: ~49,592 +- EXPECT_LE(state_size, 65536u); ++ // total: ~430,240 ++ EXPECT_LE(state_size, 512 * 1024u); + } + + //===----------------------------------------------------------------------===// +@@ -2738,6 +2738,43 @@ TEST_F(BPEModelTest, MergePriorityWithPrecedingToken) { + /*expect_pending_after_encode=*/false); + } + ++// Gemma-style tokenizers use U+2581 metaspace as a multi-byte base token. Very ++// long segments bypass the backtracking path and use the window/heap encoder, ++// which must still seed base tokens before applying rank-priority merges. ++TEST_F(BPEModelTest, LongSegmentMergePriorityAfterMultiByteBaseToken) { ++ ScopedVocabBuilder builder; ++ ++ builder.AddToken(0, "▁"); ++ builder.AddToken(1, "z"); ++ builder.AddToken(2, "m"); ++ builder.AddToken(3, "l"); ++ builder.AddToken(4, "x"); ++ builder.AddToken(5, "▁z"); ++ builder.AddToken(6, "ml"); ++ builder.AddToken(7, "▁zm"); ++ builder.AddToken(8, "zm"); ++ ++ builder.AddMerge(0, 1); // Rank 0: ▁ + z -> ▁z ++ builder.AddMerge(2, 3); // Rank 1: m + l -> ml ++ builder.AddMerge(5, 2); // Rank 2: ▁z + m -> ▁zm ++ builder.AddMerge(0, 8); // Rank 3: ▁ + zm -> ▁zm (alternate path) ++ builder.AddMerge(1, 2); // Rank 4: z + m -> zm ++ ++ CreateModel(builder); ++ ++ std::string input = "▁zml"; ++ input.append(9000, 'x'); ++ auto tokens = EncodeAndFinalize(model(), input, ++ /*expect_pending_after_encode=*/false); ++ ++ ASSERT_EQ(tokens.size(), 9002u); ++ EXPECT_EQ(tokens[0], 5); // ▁z ++ EXPECT_EQ(tokens[1], 6); // ml ++ for (size_t i = 2; i < tokens.size(); ++i) { ++ EXPECT_EQ(tokens[i], 4); ++ } ++} ++ + // Tests ByteLevel transformation of multi-byte UTF-8 sequences. + // This is the Qwen unicode bug scenario where 'él' (UTF-8: c3 a9 6c) should + // transform to 'él' (UTF-8: c3 83 c2 a9 6c) and match as a single token. +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index 2299f0de7e..76011353f9 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -1571,6 +1571,19 @@ static iree_status_t iree_tokenizer_encode_ring_as_segment( + state->transform_buffer.data, physical_write); + } + ++ if (is_partial) { ++ iree_host_size_t incomplete_tail = ++ iree_unicode_utf8_incomplete_tail_length( ++ (const char*)state->transform_buffer.data + physical_read, ++ data_length); ++ data_length -= incomplete_tail; ++ if (data_length == 0) { ++ *out_tokens_written = 0; ++ *out_segment_complete = false; ++ return iree_ok_status(); ++ } ++ } ++ + iree_tokenizer_segment_t segment = { + .start = physical_read, + .end = physical_read + data_length, +diff --git a/runtime/src/iree/tokenizer/tokenizer.h b/runtime/src/iree/tokenizer/tokenizer.h +index 098e6fef87..40db2bb459 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.h ++++ b/runtime/src/iree/tokenizer/tokenizer.h +@@ -32,16 +32,17 @@ extern "C" { + // Maximum transform buffer allocation. + // Recommended values: + // 8KB: Minimal memory, suitable for most English text +-// 16KB: Good balance for mixed content and cache locality ++// 64KB: Keeps a 32KB logical ring so large metaspace-normalized prompts can ++// stay on the exact BPE path instead of the approximate streaming path. + // + // The transform buffer determines the maximum segment size that the segmenter + // pipeline processes. Larger buffers allow longer segments, but segments that +-// exceed L1 data cache (typically 32KB) cause DFA cache misses during +-// multi-stage pre-tokenization. 16KB (8KB logical capacity after ring buffer +-// halving) keeps the working set cache-friendly for segmenters with multiple +-// children (e.g., DeepSeek V3's 4-stage Sequence pre-tokenizer). ++// exceed L1 data cache (typically 32KB) can cause DFA cache misses during ++// multi-stage pre-tokenization. 64KB (32KB logical capacity after ring buffer ++// halving) favors HuggingFace-compatible exact BPE for long normalized ++// segments. + #ifndef IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE +-#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (16 * 1024) ++#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (64 * 1024) + #endif + + // Normalizer expansion factor for transform buffer sizing. diff --git a/third_party/iree/mistral-right-aligned-numbers.patch b/third_party/iree/mistral-right-aligned-numbers.patch new file mode 100644 index 00000000..ca6e3fe5 --- /dev/null +++ b/third_party/iree/mistral-right-aligned-numbers.patch @@ -0,0 +1,303 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -225,6 +225,29 @@ + return iree_ok_status(); + } + ++ // Tekken groups numbers from the right using a positive lookahead. Match the ++ // entire number run with the DFA, then subdivide it in the Split segmenter. ++ // Restrict this lowering to the exact known pattern and isolated behavior. ++ bool right_aligned_numbers = false; ++ if (behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED && !invert && ++ iree_string_view_equal( ++ pattern, ++ IREE_SV("[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|" ++ "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/" ++ "]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"))) { ++ pattern = IREE_SV( ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}+|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"); ++ right_aligned_numbers = true; ++ } ++ + // Compile the regex pattern to a DFA. + iree_tokenizer_regex_dfa_t dfa; + uint8_t* dfa_data = NULL; +@@ -239,8 +262,13 @@ + } + + // Create the Split segmenter. On failure, free DFA data. +- status = iree_tokenizer_segmenter_split_allocate( +- dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ if (right_aligned_numbers) { ++ status = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa, dfa_data, allocator, out_segmenter); ++ } else { ++ status = iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ } + if (!iree_status_is_ok(status)) { + iree_allocator_free(allocator, dfa_data); + } +diff --git a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +--- a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel ++++ b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +@@ -188,6 +188,7 @@ + hdrs = ["split.h"], + deps = [ + "//runtime/src/iree/base", ++ "//runtime/src/iree/base/internal:unicode", + "//runtime/src/iree/tokenizer:segmenter", + "//runtime/src/iree/tokenizer/regex:exec", + ], +diff --git a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +--- a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt ++++ b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +@@ -201,6 +201,7 @@ + "split.c" + DEPS + iree::base ++ iree::base::internal::unicode + iree::tokenizer::regex::exec + iree::tokenizer::segmenter + PUBLIC +diff --git a/runtime/src/iree/tokenizer/segmenter/split.c b/runtime/src/iree/tokenizer/segmenter/split.c +--- a/runtime/src/iree/tokenizer/segmenter/split.c ++++ b/runtime/src/iree/tokenizer/segmenter/split.c +@@ -7,6 +7,8 @@ + #include "iree/tokenizer/segmenter/split.h" + + #include ++ ++#include "iree/base/internal/unicode.h" + + //===----------------------------------------------------------------------===// + // Types +@@ -19,6 +21,7 @@ + bool invert; + // false = regex mode, true = literal mode. + bool is_literal; ++ bool right_aligned_numbers; + union { + struct { + iree_tokenizer_regex_dfa_t dfa; +@@ -361,7 +364,46 @@ + iree_tokenizer_regex_split_behavior_t behavior; + bool invert; + bool absolute; ++ bool right_aligned_numbers; ++ iree_string_view_t input; + } iree_tokenizer_split_callback_context_t; ++ ++// The caller has recognized a number-only match. Count codepoints, rather than ++// bytes, so mixed Unicode number categories retain the same grouping as \p{N}. ++static iree_status_t iree_tokenizer_split_emit_number_match( ++ iree_tokenizer_split_callback_context_t* context, ++ iree_tokenizer_regex_match_t match, iree_string_view_t text) { ++ iree_host_size_t position = 0; ++ iree_host_size_t count = 0; ++ while (position < text.size) { ++ iree_unicode_utf8_decode(text, &position); ++ ++count; ++ } ++ ++ iree_tokenizer_split_emit(context->emitter, context->last_emit_end, ++ match.start, context->absolute); ++ position = 0; ++ iree_host_size_t start = match.start; ++ iree_host_size_t group_size = (count - 1) % 3 + 1; ++ while (position < text.size) { ++ for (iree_host_size_t i = 0; i < group_size; ++i) { ++ iree_unicode_utf8_decode(text, &position); ++ } ++ iree_host_size_t end = match.start + position; ++ // finalize() can retry the same match after partially filling the output. ++ if (end > context->last_emit_end) { ++ iree_tokenizer_split_emit(context->emitter, start, end, ++ context->absolute); ++ if (context->emitter->full) { ++ return iree_status_from_code(IREE_STATUS_RESOURCE_EXHAUSTED); ++ } ++ } ++ start = end; ++ group_size = 3; ++ } ++ context->last_emit_end = match.end; ++ return iree_ok_status(); ++} + + // Processes each regex match inline: computes behavior result, emits segments, + // and commits state. Returns RESOURCE_EXHAUSTED if output capacity is +@@ -370,6 +412,20 @@ + void* user_data, iree_tokenizer_regex_match_t match) { + iree_tokenizer_split_callback_context_t* context = + (iree_tokenizer_split_callback_context_t*)user_data; ++ ++ if (context->right_aligned_numbers && match.end > match.start) { ++ iree_host_size_t base = context->emitter->chunk_base; ++ if (match.start < base || match.end - base > context->input.size) { ++ return iree_make_status(IREE_STATUS_INTERNAL, ++ "number match outside retained input"); ++ } ++ iree_string_view_t text = iree_make_string_view( ++ context->input.data + match.start - base, match.end - match.start); ++ iree_host_size_t position = 0; ++ if (iree_unicode_is_number(iree_unicode_utf8_decode(text, &position))) { ++ return iree_tokenizer_split_emit_number_match(context, match, text); ++ } ++ } + + // Compute behavior result (pure, no side effects). + // Gap is the region between the last emit and the current match start. +@@ -590,6 +646,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = false; ++ segmenter->right_aligned_numbers = false; + segmenter->regex.dfa = dfa; + segmenter->regex.dfa_data = dfa_data; + +@@ -605,6 +662,17 @@ + } + IREE_TRACE_ZONE_END(z0); + return status; ++} ++ ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter) { ++ IREE_RETURN_IF_ERROR(iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, allocator, ++ out_segmenter)); ++ ((iree_tokenizer_segmenter_split_t*)*out_segmenter)->right_aligned_numbers = ++ true; ++ return iree_ok_status(); + } + + iree_status_t iree_tokenizer_segmenter_split_literal_allocate( +@@ -641,6 +709,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = true; ++ segmenter->right_aligned_numbers = false; + segmenter->literal.pattern_data = pattern_data; + segmenter->literal.pattern = + iree_make_string_view((const char*)pattern_data, pattern.size); +@@ -739,6 +808,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = input, + }; + + // Feed pattern matcher; matches are processed inline via callback. +@@ -882,6 +953,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = remaining_input, + }; + + iree_host_size_t input_end = chunk_base + remaining_input.size; +diff --git a/runtime/src/iree/tokenizer/segmenter/split.h b/runtime/src/iree/tokenizer/segmenter/split.h +--- a/runtime/src/iree/tokenizer/segmenter/split.h ++++ b/runtime/src/iree/tokenizer/segmenter/split.h +@@ -81,6 +81,14 @@ + iree_tokenizer_regex_split_behavior_t behavior, bool invert, + iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); + ++// Like ISOLATED splitting, but subdivides number-only matches into groups of ++// three Unicode numbers from the right. The DFA must match entire number runs ++// separately from other text. This implements Tekken's numeric lookahead ++// without requiring general positive lookahead support in the regex compiler. ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); ++ + // Allocates a segmenter that splits text on literal string matches. + // Unlike the regex version, this uses efficient byte-by-byte substring search. + // The |pattern| is copied and owned by the segmenter. +diff --git a/runtime/src/iree/tokenizer/segmenter/split_test.cc b/runtime/src/iree/tokenizer/segmenter/split_test.cc +--- a/runtime/src/iree/tokenizer/segmenter/split_test.cc ++++ b/runtime/src/iree/tokenizer/segmenter/split_test.cc +@@ -28,7 +28,8 @@ + public: + ScopedSplitSegmenter(const char* pattern, + iree_tokenizer_regex_split_behavior_t behavior, +- bool invert = false) { ++ bool invert = false, ++ bool right_aligned_numbers = false) { + iree_tokenizer_regex_compile_error_t error = {0}; + status_ = iree_tokenizer_regex_compile_and_load( + iree_make_cstring_view(pattern), +@@ -37,8 +38,13 @@ + if (!iree_status_is_ok(status_)) return; + + iree_tokenizer_segmenter_t* raw = nullptr; +- status_ = iree_tokenizer_segmenter_split_allocate( +- dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ if (right_aligned_numbers) { ++ status_ = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa_, dfa_storage_, iree_allocator_system(), &raw); ++ } else { ++ status_ = iree_tokenizer_segmenter_split_allocate( ++ dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ } + if (!iree_status_is_ok(status_)) return; + + segmenter_ = ScopedSegmenter(raw); +@@ -72,6 +78,44 @@ + IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED); + IREE_ASSERT_OK(segmenter.status()); + ASSERT_NE(segmenter.get(), nullptr); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ auto input = IREE_SV("a1 12 123 1234 12345 123456 1234567z"); ++ std::vector expected = { ++ "a", "1", " ", "12", " ", "123", " ", "1", "234", " ", ++ "12", "345", " ", "123", "456", " ", "1", "234", "567", "z"}; ++ TestWithAllChunkSizes(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++ TestZeroCapacityOutput(segmenter.get(), input); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedUnicodeNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ // Nd, Nl and No, including mixed UTF-8 lengths within a number run. ++ auto input = IREE_SV("١٢٣٤ 12345 Ⅷ²3४"); ++ std::vector expected = {"١", "٢٣٤", " ", "12", ++ "345", " ", "Ⅷ", "²3४"}; ++ TestWithAllChunkSizesUtf8(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbersLongRun) { ++ ScopedSplitSegmenter segmenter( ++ "\\p{N}+", IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ std::string input(1000, '1'); ++ std::vector expected(334, "111"); ++ expected[0] = "1"; ++ TestLimitedOutputCapacity(segmenter.get(), ++ iree_make_cstring_view(input.c_str()), expected); + } + + TEST(SplitSegmenterTest, StateSizeIsReasonable) { diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl new file mode 100644 index 00000000..ee89fb6d --- /dev/null +++ b/third_party/iree/repo.bzl @@ -0,0 +1,27 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +def repo(): + git_repository( + name = "iree", + remote = "https://github.com/iree-org/iree.git", + commit = "4d4e97d00f099a21f38eeff26f82a6d9e3643a11", + sparse_checkout_patterns = [ + "runtime/src/**", + "build_tools/bazel/**", + "build_tools/third_party/libbacktrace/**", + "BUILD.bazel", + ], + patches = [ + "//third_party/iree:tokenizer-only.patch", + "//third_party/iree:fix-added-token-matching.patch", + "//third_party/iree:match-hf-tokenizer.patch", + "//third_party/iree:laguna-tokenizer-compat.patch", + "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", + "//third_party/iree:mistral-right-aligned-numbers.patch", + ], + patch_args = ["-p1"], + ) + + # Use this if you want to use a local copy of IREE instead of the git repository. Make sure to update the path to point to your local IREE checkout. + #new_local_repository(name = "iree", build_file="//:third_party/iree/BUILD.bazel", path="/home/corendos/iree/") diff --git a/third_party/iree/tokenizer-only.patch b/third_party/iree/tokenizer-only.patch new file mode 100644 index 00000000..c6be27da --- /dev/null +++ b/third_party/iree/tokenizer-only.patch @@ -0,0 +1,50 @@ +diff --git a/build_tools/bazel/build_defs.oss.bzl b/build_tools/bazel/build_defs.oss.bzl +--- a/build_tools/bazel/build_defs.oss.bzl ++++ b/build_tools/bazel/build_defs.oss.bzl +@@ -4,24 +4,35 @@ + # See https://llvm.org/LICENSE.txt for license information. + # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +-load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") ++# load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") + + # All load statements must come first in Starlark. + load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +-load("@rules_shell//shell:sh_test.bzl", "sh_test") +-load( +- "//build_tools/bazel:iree_cc_fuzz.bzl", +- _iree_cc_fuzz = "iree_cc_fuzz", +- _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", +- _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", +-) ++# load("@rules_shell//shell:sh_test.bzl", "sh_test") ++# load( ++# "//build_tools/bazel:iree_cc_fuzz.bzl", ++# _iree_cc_fuzz = "iree_cc_fuzz", ++# _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", ++# _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", ++# ) + + """Common Bazel definitions for IREE.""" + + # Re-export fuzz rules for external use. +-iree_cc_fuzz = _iree_cc_fuzz +-iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz +-iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++# iree_cc_fuzz = _iree_cc_fuzz ++# iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz ++# iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++ ++def generic_pass(**kwargs): ++ pass ++ ++gentbl_cc_library = generic_pass ++gentbl_filegroup = generic_pass ++td_library = generic_pass ++sh_test = generic_pass ++iree_runtime_cc_fuzz = generic_pass ++iree_compiler_cc_fuzz = generic_pass ++iree_cc_fuzz = generic_pass + + def defaulting_select(selector): + """Pass through to select() with special semantics when converting to CMake. diff --git a/third_party/libvaxis/BUILD.bazel b/third_party/libvaxis/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/libvaxis/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch new file mode 100644 index 00000000..effbb2aa --- /dev/null +++ b/third_party/libvaxis/fixes.patch @@ -0,0 +1,202 @@ +diff --git a/src/Loop.zig b/src/Loop.zig +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -352,7 +352,7 @@ + } + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -484,7 +484,7 @@ + vx.caps.program_status = true; + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { +diff --git a/src/Vaxis.zig b/src/Vaxis.zig +--- a/src/Vaxis.zig ++++ b/src/Vaxis.zig +@@ -139,14 +139,13 @@ + } + } + +-/// resets enabled features, sends cursor to home and clears below cursor +-pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { +- // always show the cursor on state reset ++/// Resets enabled terminal features without clearing screen content. ++/// Use this for non-interactive / single-shot rendering where the ++/// output should remain visible after exit. ++pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { + tty.writeAll(ctlseqs.show_cursor) catch {}; + tty.writeAll(ctlseqs.sgr_reset) catch {}; + if (self.screen.cursor_shape != .default) { +- // In many terminals, `.default` will set to the configured cursor shape. Others, it will +- // change to a blinking block. + tty.print(ctlseqs.cursor_shape, .{@backingInt(Cell.CursorShape.default)}) catch {}; + } + if (self.state.kitty_keyboard) { +@@ -159,6 +158,32 @@ + if (self.state.bracketed_paste) { + try self.setBracketedPaste(tty, false); + } ++ if (self.state.color_scheme_updates) { ++ try tty.writeAll(ctlseqs.color_scheme_reset); ++ self.state.color_scheme_updates = false; ++ } ++ if (self.state.in_band_resize) { ++ try tty.writeAll(ctlseqs.in_band_resize_reset); ++ self.state.in_band_resize = false; ++ } ++ if (self.state.changed_default_fg) { ++ try tty.writeAll(ctlseqs.osc10_reset); ++ self.state.changed_default_fg = false; ++ } ++ if (self.state.changed_default_bg) { ++ try tty.writeAll(ctlseqs.osc11_reset); ++ self.state.changed_default_bg = false; ++ } ++ if (self.state.changed_cursor_color) { ++ try tty.writeAll(ctlseqs.osc12_reset); ++ self.state.changed_cursor_color = false; ++ } ++ try tty.flush(); ++} ++ ++/// resets enabled features, sends cursor to home and clears below cursor ++pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { ++ self.resetModes(tty) catch {}; + if (self.state.alt_screen) { + try tty.writeAll(ctlseqs.home); + try tty.writeAll(ctlseqs.erase_below_cursor); +@@ -171,27 +196,6 @@ + } + try tty.writeAll(ctlseqs.erase_below_cursor); + } +- if (self.state.color_scheme_updates) { +- try tty.writeAll(ctlseqs.color_scheme_reset); +- self.state.color_scheme_updates = false; +- } +- if (self.state.in_band_resize) { +- try tty.writeAll(ctlseqs.in_band_resize_reset); +- self.state.in_band_resize = false; +- } +- if (self.state.changed_default_fg) { +- try tty.writeAll(ctlseqs.osc10_reset); +- self.state.changed_default_fg = false; +- } +- if (self.state.changed_default_bg) { +- try tty.writeAll(ctlseqs.osc11_reset); +- self.state.changed_default_bg = false; +- } +- if (self.state.changed_cursor_color) { +- try tty.writeAll(ctlseqs.osc12_reset); +- self.state.changed_cursor_color = false; +- } +- + try tty.flush(); + } + +@@ -1101,19 +1105,19 @@ + switch (format) { + .rgb => { + try tty.print( +- "\x1b_Gf=24,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=24,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .rgba => { + try tty.print( +- "\x1b_Gf=32,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=32,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .png => { + try tty.print( +- "\x1b_Gf=100,i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=100,i={d},t={c},q=2;{s}\x1b\\", + .{ id, medium_char, encoded }, + ); + }, +@@ -1149,7 +1153,7 @@ + + if (bytes.len < 4096) { + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d};{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},q=2;{s}\x1b\\", + .{ + fmt, + width, +@@ -1162,14 +1166,14 @@ + var n: usize = 4096; + + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d},m=1;{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},m=1,q=2;{s}\x1b\\", + .{ fmt, width, height, id, bytes[0..n] }, + ); + while (n < bytes.len) : (n += 4096) { + const end: usize = @min(n + 4096, bytes.len); + const m: u2 = if (end == bytes.len) 0 else 1; + try tty.print( +- "\x1b_Gm={d};{s}\x1b\\", ++ "\x1b_Gm={d},q=2;{s}\x1b\\", + .{ + m, + bytes[n..end], +diff --git a/src/c_api.zig b/src/c_api.zig +--- a/src/c_api.zig ++++ b/src/c_api.zig +@@ -965,7 +965,7 @@ + .cap_multi_cursor => vx.caps.multi_cursor = true, + .cap_program_status => vx.caps.program_status = true, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => { +diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig +--- a/src/ctlseqs.zig ++++ b/src/ctlseqs.zig +@@ -140,8 +140,8 @@ + pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; + + // Kitty graphics +-pub const kitty_graphics_clear = "\x1b_Ga=d\x1b\\"; +-pub const kitty_graphics_preamble = "\x1b_Ga=p,i={d}"; ++pub const kitty_graphics_clear = "\x1b_Ga=d,q=2\x1b\\"; ++pub const kitty_graphics_preamble = "\x1b_Ga=p,q=2,i={d}"; + pub const kitty_graphics_closing = ",C=1\x1b\\"; + + // Color control sequences +diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig +--- a/src/vxfw/Padding.zig ++++ b/src/vxfw/Padding.zig +@@ -6,7 +6,7 @@ + const vxfw = @import("vxfw.zig"); + + const Padding = @This(); +-const PadValues = struct { ++pub const PadValues = struct { + left: u16 = 0, + right: u16 = 0, + top: u16 = 0, +@@ -54,10 +54,6 @@ + + pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const pad = self.padding; +- if (pad.left > 0 or pad.right > 0) +- std.debug.assert(ctx.max.width != null); +- if (pad.top > 0 or pad.bottom > 0) +- std.debug.assert(ctx.max.height != null); + const inner_min: vxfw.Size = .{ + .width = ctx.min.width -| (pad.right + pad.left), + .height = ctx.min.height -| (pad.top + pad.bottom), diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel b/third_party/libvaxis/libvaxis.bazel similarity index 51% rename from third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel rename to third_party/libvaxis/libvaxis.bazel index 872ef7fb..122e25a1 100644 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel +++ b/third_party/libvaxis/libvaxis.bazel @@ -1,13 +1,17 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "xev", + name = "vaxis", srcs = glob([ "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", + "src/vxfw/*.zig", + "src/widgets/*.zig", + "src/widgets/terminal/*.zig", ]), main = "src/main.zig", + deps = [ + "@zigimg", + "@uucode", + ], visibility = ["//visibility:public"], ) diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl new file mode 100644 index 00000000..e10982a4 --- /dev/null +++ b/third_party/libvaxis/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "libvaxis", + remote = "https://github.com/rockorager/libvaxis.git", + commit = "14b7684272234a3d66316c0d1bbdc876d4e8997c", + build_file = "//third_party/libvaxis:libvaxis.bazel", + patches = ["//third_party/libvaxis:fixes.patch"], + patch_args = ["-p1"], + ) diff --git a/third_party/linenoise/repo.bzl b/third_party/linenoise/repo.bzl index 7837ac67..1d7a1678 100644 --- a/third_party/linenoise/repo.bzl +++ b/third_party/linenoise/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "linenoise", remote = "https://github.com/antirez/linenoise.git", commit = "dc83cc373ac2058030eb3cf5e404959a26fef112", diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel b/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel deleted file mode 100644 index 7f1750b4..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel +++ /dev/null @@ -1,78 +0,0 @@ -"aspect-build/bazel-lib" - -module( - name = "aspect_bazel_lib", - version = "2.8.1.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, -) - -# Lower-bounds (minimum) versions for direct runtime dependencies -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "platforms", version = "0.0.8") - -# 0.5.4 is the first version with bzlmod support -bazel_dep(name = "stardoc", version = "0.6.2", repo_name = "io_bazel_stardoc") - -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") -bazel_lib_toolchains.copy_directory() -bazel_lib_toolchains.copy_to_directory() -bazel_lib_toolchains.jq() -bazel_lib_toolchains.yq() -bazel_lib_toolchains.coreutils() -bazel_lib_toolchains.tar() -bazel_lib_toolchains.zstd() -bazel_lib_toolchains.expand_template() -bazel_lib_toolchains.bats() -use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains") - -register_toolchains( - "@copy_directory_toolchains//:all", - "@copy_to_directory_toolchains//:all", - "@jq_toolchains//:all", - "@yq_toolchains//:all", - "@coreutils_toolchains//:all", - "@expand_template_toolchains//:all", - "@bats_toolchains//:all", - "@bsd_tar_toolchains//:all", - "@zstd_toolchains//:all", -) - -####### Dev dependencies ######## - -# To allow /tools to be built from source -# NOTE: when publishing to BCR, we patch this to be dev_dependency, as we publish pre-built binaries -# along with our releases. - -bazel_dep( - name = "gazelle", - version = "0.36.0", - dev_dependency = True, -) -bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", - dev_dependency = True, -) - -go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", - dev_dependency = True, -) -go_deps.from_file(go_mod = "//:go.mod") -use_repo( - go_deps, - "com_github_bmatcuk_doublestar_v4", - "org_golang_x_exp", - "org_golang_x_sys", -) - -host = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "host", dev_dependency = True) -host.host() -use_repo(host, "aspect_bazel_lib_host") - -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True) -bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) -bazel_dep(name = "bazel_features", version = "0.2.0", dev_dependency = True) diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch deleted file mode 100644 index 96aa8dfc..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 5dbe6a2604e440b684add7b44531898acc8631b4 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Sun, 8 Sep 2024 21:17:29 +0200 -Subject: [PATCH] fix: add repo mapping to tar archive - -When using bzlmod, runfiles lookup will fail without it. ---- - lib/private/tar.bzl | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl -index 733ff60..29434f6 100644 ---- a/lib/private/tar.bzl -+++ b/lib/private/tar.bzl -@@ -147,12 +147,19 @@ def _tar_impl(ctx): - args.add(ctx.file.mtree, format = "@%s") - inputs.append(ctx.file.mtree) - -+ src_runfiles = [] -+ for src in ctx.attr.srcs: -+ src_di = src[DefaultInfo] -+ if getattr(src_di.files_to_run, "repo_mapping_manifest", None) != None: -+ src_runfiles.append(depset( -+ direct = [src_di.files_to_run.repo_mapping_manifest], -+ transitive = [src_di.default_runfiles.files], -+ )) -+ else: -+ src_runfiles.append(src_di.default_runfiles.files) - ctx.actions.run( - executable = bsdtar.tarinfo.binary, -- inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + [ -- src[DefaultInfo].default_runfiles.files -- for src in ctx.attr.srcs -- ]), -+ inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + src_runfiles), - outputs = [out], - arguments = [args], - mnemonic = "Tar", -@@ -234,6 +241,8 @@ def _mtree_impl(ctx): - workspace_name = str(ctx.workspace_name) - - content.add(_mtree_line(runfiles_dir, type = "dir")) -+ if getattr(default_info.files_to_run, "repo_mapping_manifest", None) != None: -+ content.add(_mtree_line("{}/_repo_mapping".format(runfiles_dir), type = "file", content = default_info.files_to_run.repo_mapping_manifest.path)) - content.add_all( - s.default_runfiles.files, - expand_directories = True, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch deleted file mode 100644 index ffe1c1d3..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/MODULE.bazel b/MODULE.bazel -index e63fa5b..9d78a88 100644 ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -50,19 +50,19 @@ use_repo(host, "aspect_bazel_lib_host") - bazel_dep( - name = "gazelle", - version = "0.36.0", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - - go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - go_deps.from_file(go_mod = "//:go.mod") - use_repo( diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch deleted file mode 100644 index fe11475e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch +++ /dev/null @@ -1,14 +0,0 @@ -=================================================================== ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,9 +1,9 @@ - "aspect-build/bazel-lib" - - module( - name = "aspect_bazel_lib", -- version = "0.0.0", -+ version = "2.8.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - ) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json b/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json deleted file mode 100644 index 3b7d500e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "integrity": "sha256-aINU7mvuunGUJD1z6wmSuaEujt7u7FtlRPS1MaMRIjc=", - "strip_prefix": "bazel-lib-2.8.1", - "url": "https://github.com/aspect-build/bazel-lib/releases/download/v2.8.1/bazel-lib-v2.8.1.tar.gz", - "patches": { - "go_dev_dep.patch": "sha256-DTc/hk+etl4D50M0BLRik2vHbrgDb6rds+Dj4xphWb4=", - "module_dot_bazel_version.patch": "sha256-cEMv6bY7Sc5dERv8YG0lq45zuZCsVPNn4oAN9aOkf40=", - "0001-fix-add-repo-mapping-to-tar-archive.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/aspect_bazel_lib/metadata.json b/third_party/modules/aspect_bazel_lib/metadata.json deleted file mode 100644 index db6bde15..00000000 --- a/third_party/modules/aspect_bazel_lib/metadata.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "homepage": "https://docs.aspect.build/rules/aspect_bazel_lib", - "maintainers": [ - { - "name": "Alex Eagle", - "email": "alex@aspect.dev", - "github": "alexeagle" - }, - { - "name": "Derek Cormier", - "email": "derek@aspect.dev", - "github": "kormide" - } - ], - "repository": [ - "github:aspect-build/bazel-lib" - ], - "versions": [ - "2.8.1.1" - ], - "yanked_versions": { - "1.31.0": "1.31.0 has a breaking change to the default yq version", - "1.34.2": "The version field has a leading 'v' due to a release automation bug" - } -} diff --git a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel deleted file mode 100644 index 8f8806c6..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240825.0-dbe2291", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240825.0-dbe2291/source.json b/third_party/modules/libxev/20240825.0-dbe2291/source.json deleted file mode 100644 index 81950288..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-dbe22910a43e9e8ec9948d3cbd73d8488a074967", - "url": "https://github.com/mitchellh/libxev/archive/dbe22910a43e9e8ec9948d3cbd73d8488a074967.tar.gz", - "integrity": "sha256-JM70bkRUyuAQXKV1piJ6I0IClzCrXV43sdLfIKVC9Lo=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel deleted file mode 100644 index 99daae70..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240910.0-a2d9b31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json deleted file mode 100644 index c56bc6a8..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-a2d9b3154f1e5ed463c9f2fb98a0d739057796ce", - "url": "https://github.com/zml/libxev/archive/a2d9b3154f1e5ed463c9f2fb98a0d739057796ce.tar.gz", - "integrity": "sha256-xdMjrGSB3telt52iRqHgem0WSRPjfGlTKBRyDMXLlaQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel deleted file mode 100644 index c9b03066..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241119.0-6afcde9", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig b/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch b/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch deleted file mode 100644 index 6643419d..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch +++ /dev/null @@ -1,119 +0,0 @@ -From 0d1c2f8258072148459d3114b9ccaf43c02e0958 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - diff --git a/third_party/modules/libxev/20241119.0-6afcde9/source.json b/third_party/modules/libxev/20241119.0-6afcde9/source.json deleted file mode 100644 index de3c2417..00000000 --- a/third_party/modules/libxev/20241119.0-6afcde9/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-690c76fd792f001c5776716f1e7b04be2cc50b52", - "url": "https://github.com/zml/libxev/archive/690c76fd792f001c5776716f1e7b04be2cc50b52.tar.gz", - "integrity": "sha256-DV66ic8PcRnG3EdimswCleiHo/dDztgebz/1EY5XDXg=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch deleted file mode 100644 index 6db29cc8..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch +++ /dev/null @@ -1,348 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.0-db6a52b/source.json b/third_party/modules/libxev/20241208.0-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.0-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel deleted file mode 100644 index ea8c55d7..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.1-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch deleted file mode 100644 index 361bc9d1..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch +++ /dev/null @@ -1,323 +0,0 @@ -From 8927108937e23ce2ad547a310dabd976ddcaa39d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - - -From bf6c19eb1c06d1fb1b3cb5a66ea2ce8f743da5ba Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - -From c0ae753a9874bf78a48bf25b4362b6cca4de1da1 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + - -From 84e75a10ec1c10ea2a13edfcd8a31e582458fc88 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/watcher/stream.zig | 16 ++++++++++++++++ - 2 files changed, 29 insertions(+), 2 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; - -From 5bc3ad9f23a03ff3af7a4ec33c8bfa4f3cc6bae9 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.1-db6a52b/source.json b/third_party/modules/libxev/20241208.1-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.1-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel deleted file mode 100644 index 1d593b7d..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.2-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel deleted file mode 100644 index ce556ab1..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20241208.0-db6a52b", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch deleted file mode 100644 index 18349433..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch +++ /dev/null @@ -1,359 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - --- -2.39.5 (Apple Git-154) - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - --- -2.39.5 (Apple Git-154) - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + --- -2.39.5 (Apple Git-154) - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; --- -2.39.5 (Apple Git-154) - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/libxev/20241208.2-db6a52b/source.json b/third_party/modules/libxev/20241208.2-db6a52b/source.json deleted file mode 100644 index f08c9517..00000000 --- a/third_party/modules/libxev/20241208.2-db6a52b/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", - "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", - "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel deleted file mode 100644 index 6102ba67..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250124.0-31eed4e", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel deleted file mode 100644 index 7c0c4ce9..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "main2.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig b/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig deleted file mode 100644 index 4166925a..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig +++ /dev/null @@ -1,22 +0,0 @@ -const builtin = @import("builtin"); -const root = @import("root"); - -const main = @import("src/main.zig"); - -pub const ThreadPool = main.ThreadPool; -pub const stream = main.stream; - -pub const Options = struct { - linux_backend: main.Backend = .epoll, -}; - -pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; - -const default: main.Backend = switch (builtin.os.tag) { - .ios, .macos => .kqueue, - .linux => options.linux_backend, - .wasi => .wasi_poll, - .windows => .iocp, - else => @compileError("Unsupported OS"), -}; -pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch deleted file mode 100644 index 18349433..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch +++ /dev/null @@ -1,359 +0,0 @@ -From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 19 Nov 2024 16:14:14 +0100 -Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification - -Tries to mimic what happens in backend/kqueue. - -Closes #4 ---- - src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 42 insertions(+) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index ae4ec7d..f44d326 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -21,6 +21,12 @@ pub const Loop = struct { - - fd: posix.fd_t, - -+ /// The eventfd that this epoll queue always has a filter for. Writing -+ /// an empty message to this eventfd can be used to wake up the loop -+ /// at any time. Waking up the loop via this eventfd won't trigger any -+ /// particular completion, it just forces tick to cycle. -+ eventfd: xev.Async, -+ - /// The number of active completions. This DOES NOT include completions that - /// are queued in the submissions queue. - active: usize = 0, -@@ -56,8 +62,12 @@ pub const Loop = struct { - } = .{}, - - pub fn init(options: xev.Options) !Loop { -+ var eventfd = try xev.Async.init(); -+ errdefer eventfd.deinit(); -+ - var res: Loop = .{ - .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), -+ .eventfd = eventfd, - .thread_pool = options.thread_pool, - .thread_pool_completions = undefined, - .cached_now = undefined, -@@ -68,6 +78,7 @@ pub const Loop = struct { - - pub fn deinit(self: *Loop) void { - posix.close(self.fd); -+ self.eventfd.deinit(); - } - - /// Run the event loop. See RunMode documentation for details on modes. -@@ -262,9 +273,26 @@ pub const Loop = struct { - // Initialize - if (!self.flags.init) { - self.flags.init = true; -+ - if (self.thread_pool != null) { - self.thread_pool_completions.init(); - } -+ -+ var ev: linux.epoll_event = .{ -+ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -+ .data = .{ .ptr = 0 }, -+ }; -+ posix.epoll_ctl( -+ self.fd, -+ linux.EPOLL.CTL_ADD, -+ self.eventfd.fd, -+ &ev, -+ ) catch |err| { -+ // We reset initialization because we can't do anything -+ // safely unless we get this mach port registered! -+ self.flags.init = false; -+ return err; -+ }; - } - - // Submit all the submissions. We copy the submission queue so that -@@ -369,6 +397,10 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -+ // Zero data values are internal events that we do nothing -+ // on such as the eventfd wakeup. -+ if (ev.data.ptr == 0) continue; -+ - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - - // We get the fd and mark this as in progress we can properly -@@ -415,6 +447,7 @@ pub const Loop = struct { - const pool = self.thread_pool orelse return error.ThreadPoolRequired; - - // Setup our completion state so that thread_perform can do stuff -+ c.task_loop = self; - c.task_completions = &self.thread_pool_completions; - c.task = .{ .callback = Loop.thread_perform }; - -@@ -436,6 +469,14 @@ pub const Loop = struct { - - // Add to our completion queue - c.task_completions.push(c); -+ -+ // Wake up our main loop -+ c.task_loop.wakeup() catch {}; -+ } -+ -+ /// Sends an empty message to this loop's eventfd so that it wakes up. -+ fn wakeup(self: *Loop) !void { -+ try self.eventfd.notify(); - } - - fn start(self: *Loop, completion: *Completion) void { -@@ -800,6 +841,7 @@ pub const Completion = struct { - /// reliable way to get access to the loop and shouldn't be used - /// except internally. - task: ThreadPool.Task = undefined, -+ task_loop: *Loop = undefined, - task_completions: *Loop.TaskCompletionQueue = undefined, - task_result: Result = undefined, - --- -2.39.5 (Apple Git-154) - -From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Tue, 14 Jan 2025 14:43:54 +0000 -Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being - awaken again - ---- - src/backend/epoll.zig | 11 +++++++---- - 1 file changed, 7 insertions(+), 4 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f44d326..f84c687 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -280,7 +280,7 @@ pub const Loop = struct { - - var ev: linux.epoll_event = .{ - .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, -- .data = .{ .ptr = 0 }, -+ .data = .{ .fd = self.eventfd.fd }, - }; - posix.epoll_ctl( - self.fd, -@@ -397,9 +397,12 @@ pub const Loop = struct { - - // Process all our events and invoke their completion handlers - for (events[0..n]) |ev| { -- // Zero data values are internal events that we do nothing -- // on such as the eventfd wakeup. -- if (ev.data.ptr == 0) continue; -+ // Handle wakeup eventfd -+ if (ev.data.fd == self.eventfd.fd) { -+ var buffer: u64 = undefined; -+ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; -+ continue; -+ } - - const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); - --- -2.39.5 (Apple Git-154) - -From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:47:42 +0000 -Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait - -Since eventfd is now implemented. ---- - src/backend/epoll.zig | 4 +--- - 1 file changed, 1 insertion(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index f84c687..e3eee20 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -375,9 +375,7 @@ pub const Loop = struct { - const timeout: i32 = if (wait_rem == 0) 0 else timeout: { - // If we have a timer, we want to set the timeout to our next - // timer value. If we have no timer, we wait forever. -- // TODO: do not wait 100ms here, use an eventfd for our -- // thread pool to wake us up. -- const t = self.timers.peek() orelse break :timeout 100; -+ const t = self.timers.peek() orelse break :timeout -1; - - // Determine the time in milliseconds. - const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + --- -2.39.5 (Apple Git-154) - -From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:48:27 +0000 -Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool - -Close might block, so dispatch it inside a threadpool. ---- - src/backend/epoll.zig | 15 +++++++++++++-- - src/backend/kqueue.zig | 6 +++++- - src/watcher/stream.zig | 16 ++++++++++++++++ - 3 files changed, 34 insertions(+), 3 deletions(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index e3eee20..0f4a2ac 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -700,6 +700,13 @@ pub const Loop = struct { - }, - - .close => |v| res: { -+ if (completion.flags.threadpool) { -+ if (self.thread_schedule(completion)) |_| -+ return -+ else |err| -+ break :res .{ .close = err }; -+ } -+ - posix.close(v.fd); - break :res .{ .close = {} }; - }, -@@ -909,7 +916,6 @@ pub const Completion = struct { - // This should never happen because we always do these synchronously - // or in another location. - .cancel, -- .close, - .noop, - .shutdown, - .timer, -@@ -1015,6 +1021,11 @@ pub const Completion = struct { - err, - }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ - Unknown, - }; - --pub const CloseError = posix.EpollCtlError || error{ -+pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ - Unknown, - }; - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..9914309 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -1102,7 +1102,6 @@ pub const Completion = struct { - fn perform(self: *Completion, ev_: ?*const Kevent) Result { - return switch (self.op) { - .cancel, -- .close, - .noop, - .timer, - .shutdown, -@@ -1232,6 +1231,11 @@ pub const Completion = struct { - - break :res .{ .proc = 0 }; - }, -+ -+ .close => |*op| res: { -+ posix.close(op.fd); -+ break :res .{ .close = {} }; -+ }, - }; - } - -diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig -index 7f5df6f..bc95282 100644 ---- a/src/watcher/stream.zig -+++ b/src/watcher/stream.zig -@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options - }).callback, - }; - -+ // If we're dup-ing, then we ask the backend to manage the fd. -+ switch (xev.backend) { -+ .io_uring, -+ .wasi_poll, -+ .iocp, -+ => {}, -+ -+ .epoll => { -+ c.flags.threadpool = true; -+ }, -+ -+ .kqueue => { -+ c.flags.threadpool = true; -+ }, -+ } -+ - loop.add(c); - } - }; --- -2.39.5 (Apple Git-154) - -From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Fri, 17 Jan 2025 20:59:18 +0000 -Subject: [PATCH 5/5] epoll: don't count immediate actions - -If an immediate action is dispatched, the loop might block -on epoll_wait even though only one action was requested. ---- - src/backend/epoll.zig | 15 ++++++++++++++- - 1 file changed, 14 insertions(+), 1 deletion(-) - -diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig -index 0f4a2ac..fb7e59e 100644 ---- a/src/backend/epoll.zig -+++ b/src/backend/epoll.zig -@@ -297,6 +297,7 @@ pub const Loop = struct { - - // Submit all the submissions. We copy the submission queue so that - // any resubmits don't cause an infinite loop. -+ var wait_rem: usize = @intCast(wait); - var queued = self.submissions; - self.submissions = .{}; - while (queued.pop()) |c| { -@@ -304,6 +305,19 @@ pub const Loop = struct { - // This usually means that we switched them to be deleted or - // something. - if (c.flags.state != .adding) continue; -+ -+ // These operations happen synchronously. Ensure they are -+ // decremented from wait_rem. -+ switch (c.op) { -+ .cancel, -+ // should noop be counted? -+ // .noop, -+ .shutdown, -+ .timer, -+ => wait_rem -|= 1, -+ else => {}, -+ } -+ - self.start(c); - } - -@@ -322,7 +336,6 @@ pub const Loop = struct { - - // Wait and process events. We only do this if we have any active. - var events: [1024]linux.epoll_event = undefined; -- var wait_rem: usize = @intCast(wait); - while (self.active > 0 and (wait == 0 or wait_rem > 0)) { - self.update_now(); - const now_timer: Operation.Timer = .{ .next = self.cached_now }; --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch deleted file mode 100644 index 70cee31c..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 283a911561385e0cc9278dfd350bdc5c5132c9c4 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Wed, 5 Feb 2025 11:17:01 +0100 -Subject: [PATCH] backend/kqueue: fix empty udata crash - -On certain occasions, internal kqueue events are dispatched with an empty udata field. -This causes a crash when trying to access the udata field in the event handler. - -This patch adds a check to ensure that the udata field is not empty before accessing it -just like in the other events consumer. ---- - src/backend/kqueue.zig | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig -index 456f64f..7d919ce 100644 ---- a/src/backend/kqueue.zig -+++ b/src/backend/kqueue.zig -@@ -206,11 +206,15 @@ pub const Loop = struct { - // event list to zero length) because it was leading to - // memory corruption we need to investigate. - for (events[0..completed]) |ev| { -- const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); -+ // Zero udata values are internal events that we do nothing -+ // on such as the mach port wakeup. -+ if (ev.udata == 0) continue; - - // We handle deletions separately. - if (ev.flags & posix.system.EV_DELETE != 0) continue; - -+ const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); -+ - // If EV_ERROR is set, then submission failed for this - // completion. We get the syscall errorcode from data and - // store it. diff --git a/third_party/modules/libxev/20250124.0-31eed4e/source.json b/third_party/modules/libxev/20250124.0-31eed4e/source.json deleted file mode 100644 index 77af3d8f..00000000 --- a/third_party/modules/libxev/20250124.0-31eed4e/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "libxev-31eed4e337fed7b0149319e5cdbb62b848c24fbd", - "url": "https://github.com/zml/libxev/archive/31eed4e337fed7b0149319e5cdbb62b848c24fbd.tar.gz", - "integrity": "sha256-VHP90NTytIZ8UZsYRKOOxN490/I6yv6ec40sP8y5MJ8=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "", - "main2.zig": "" - }, - "patches": { - "128.patch": "", - "141.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel deleted file mode 100644 index 424792ed..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250222.0-07bcffa", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250222.0-07bcffa/source.json b/third_party/modules/libxev/20250222.0-07bcffa/source.json deleted file mode 100644 index 1483ad30..00000000 --- a/third_party/modules/libxev/20250222.0-07bcffa/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-07bcffa0f63054152a9883baa42bce5faad297e6", - "url": "https://github.com/mitchellh/libxev/archive/07bcffa0f63054152a9883baa42bce5faad297e6.tar.gz", - "integrity": "sha256-H80IoKcTeJFwpL5i4S9zyMuyWaQCK173ppoAO/CYXjc=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel deleted file mode 100644 index bfe80aeb..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250313.0-5773f46", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250313.0-5773f46/source.json b/third_party/modules/libxev/20250313.0-5773f46/source.json deleted file mode 100644 index 65786582..00000000 --- a/third_party/modules/libxev/20250313.0-5773f46/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "libxev-5773f46de3937e848db3ff87577add8ef7e8222d", - "url": "https://github.com/mitchellh/libxev/archive/5773f46de3937e848db3ff87577add8ef7e8222d.tar.gz", - "integrity": "sha256-k9cyG0DPhEKwM+/dKiLFT9WeM+c2kCokN8VftOriSxI=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel deleted file mode 120000 index c8a52909..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel deleted file mode 100644 index 82907dd7..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20250718.0-9f785d2", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") diff --git a/third_party/modules/libxev/20250718.0-9f785d2/source.json b/third_party/modules/libxev/20250718.0-9f785d2/source.json deleted file mode 100644 index daf74448..00000000 --- a/third_party/modules/libxev/20250718.0-9f785d2/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", - "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", - "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} - diff --git a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel deleted file mode 120000 index c8a52909..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel deleted file mode 100644 index 66f5a554..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20251010.0-9f785d2", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "0.12.1") diff --git a/third_party/modules/libxev/20251010.0-9f785d2/source.json b/third_party/modules/libxev/20251010.0-9f785d2/source.json deleted file mode 100644 index daf74448..00000000 --- a/third_party/modules/libxev/20251010.0-9f785d2/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", - "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", - "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1 -} - diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json deleted file mode 100644 index 1fd34d18..00000000 --- a/third_party/modules/libxev/metadata.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "homepage": "https://github.com/mitchellh/libxev", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:mitchellh/libxev" - ], - "versions": [ - "20240825.0-dbe2291", - "20240910.0-a2d9b31", - "20241119.0-6afcde9", - "20241208.0-db6a52b", - "20241208.1-db6a52b", - "20241208.2-db6a52b", - "20250124.0-31eed4e", - "20250222.0-07bcffa", - "20250313.0-5773f46", - "20250716.0-9f785d2", - "20251010.0-9f785d2" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel deleted file mode 100644 index 6bf421a8..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240823.0-f142f8a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json b/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json deleted file mode 100644 index 6edc73dc..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-f142f8afe21bceb00fb495468aa0b5043e98c419", - "url": "https://github.com/llvm/llvm-project/archive/f142f8afe21bceb00fb495468aa0b5043e98c419.tar.gz", - "integrity": "sha256-8ftYOnMGq99igt2I1n9SOZpYLq79+F7HiKL/7lOwcHs=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel deleted file mode 100644 index c9130b80..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240919.0-94c024a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json b/third_party/modules/llvm-raw/20240919.0-94c024a/source.json deleted file mode 100644 index 2aec201a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-94c024adedcb53059c29d7c2d62982053b60e86a", - "url": "https://github.com/llvm/llvm-project/archive/94c024adedcb53059c29d7c2d62982053b60e86a.tar.gz", - "integrity": "sha256-IEzt6quG8GXvZMs4id0ukt3UqPXVtrwctLJ2aU+2p5g=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel deleted file mode 100644 index b3ed250a..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20241022.0-6c4267f", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json b/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json deleted file mode 100644 index f2286dc8..00000000 --- a/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-6c4267fb1779bc5550bb413f33250f9365acfbc6", - "url": "https://github.com/llvm/llvm-project/archive/6c4267fb1779bc5550bb413f33250f9365acfbc6.tar.gz", - "integrity": "sha256-cBDuj+hiRvq8rtvtIfqawr0lQuDSrWFypEgeApT981Q=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel deleted file mode 100644 index 102b0a38..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250102.0-f739aa4", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json b/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json deleted file mode 100644 index 146f68ab..00000000 --- a/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-f739aa4004165dc64d3a1f418d5ad3c84886f01a", - "url": "https://github.com/llvm/llvm-project/archive/f739aa4004165dc64d3a1f418d5ad3c84886f01a.tar.gz", - "integrity": "sha256-hdoTTnugRO9Q68AJ0aV6h/sOLbIIoEZQ7y/lZOlWSqc=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel deleted file mode 100644 index b1171a0a..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250117.0-bf17016", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/source.json b/third_party/modules/llvm-raw/20250117.0-bf17016/source.json deleted file mode 100644 index 851e85aa..00000000 --- a/third_party/modules/llvm-raw/20250117.0-bf17016/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-bf17016a92bc8a23d2cdd2b51355dd4eb5019c68", - "url": "https://github.com/llvm/llvm-project/archive/bf17016a92bc8a23d2cdd2b51355dd4eb5019c68.tar.gz", - "integrity": "sha256-ugnxLlAZ9aylMbFzMnXwoQsYHW+JTesaRhDgF/drFyo=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel deleted file mode 100644 index c213afe0..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250201.0-386af4a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/source.json b/third_party/modules/llvm-raw/20250201.0-386af4a/source.json deleted file mode 100644 index e3a7a1ac..00000000 --- a/third_party/modules/llvm-raw/20250201.0-386af4a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-386af4a5c64ab75eaee2448dc38f2e34a40bfed0", - "url": "https://github.com/llvm/llvm-project/archive/386af4a5c64ab75eaee2448dc38f2e34a40bfed0.tar.gz", - "integrity": "sha256-S3UqLfteTlXPmBKjU7zn5DfghS9O72SSn0jXRzIIDFA=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel deleted file mode 100644 index 245f034b..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel +++ /dev/null @@ -1,11 +0,0 @@ -module( - name = "llvm-raw", - version = "20250217.0-0e779ad", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.11") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") -bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json b/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json deleted file mode 100644 index 6add20e8..00000000 --- a/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-0e779ad4998ef65907502101c5b82ede05ddfa4e", - "url": "https://github.com/llvm/llvm-project/archive/0e779ad4998ef65907502101c5b82ede05ddfa4e.tar.gz", - "integrity": "sha256-1cJWCy2c487XlRET8rXR6kKGZWePTcsfuHgOsSGcphU=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/metadata.json b/third_party/modules/llvm-raw/metadata.json deleted file mode 100644 index 32713e58..00000000 --- a/third_party/modules/llvm-raw/metadata.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "homepage": "https://github.com/llvm/llvm-project", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:llvm/llvm-project" - ], - "versions": [ - "20240823.0-f142f8a", - "20240919.0-94c024a", - "20241022.0-6c4267f", - "20250102.0-f739aa4", - "20250117.0-bf17016", - "20250201.0-386af4a", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel b/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel deleted file mode 100644 index 723c6a98..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240904.0-010da15", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240904.0-010da15/source.json b/third_party/modules/rules_zig/20240904.0-010da15/source.json deleted file mode 100644 index 253af196..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "010da15abb4335479778d6b4fb2ca752a0ab80e3" -} diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel b/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel deleted file mode 100644 index 4ded2bb6..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240909.0-37f17ff", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json b/third_party/modules/rules_zig/20240909.0-37f17ff/source.json deleted file mode 100644 index 19d05aef..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "37f17ffd2f6d04fc4bf9b583f9a1b117856a88d1" -} diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel b/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel deleted file mode 100644 index e288be56..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240912.0-41bfe84", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json b/third_party/modules/rules_zig/20240912.0-41bfe84/source.json deleted file mode 100644 index 3d576fb1..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "41bfe84e4d9a43cbe55281dddc80b683cc6fc6eb" -} diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel b/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel deleted file mode 100644 index f2ff768e..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240913.0-1957d05", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/source.json b/third_party/modules/rules_zig/20240913.0-1957d05/source.json deleted file mode 100644 index 9da3ae38..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "1957d0572193fb859e721a0fab8bd8f0fb57f3ff" -} diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel b/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel deleted file mode 100644 index 392e8aff..00000000 --- a/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250314.0-b9739c6", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/source.json b/third_party/modules/rules_zig/20250314.0-b9739c6/source.json deleted file mode 100644 index ceea72c3..00000000 --- a/third_party/modules/rules_zig/20250314.0-b9739c6/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-b9739c615ce62b64ee595ac4bcd9ee7cc06b0422", - "url": "https://github.com/zml/rules_zig/archive/b9739c615ce62b64ee595ac4bcd9ee7cc06b0422.tar.gz", - "integrity": "sha256-p1rYD9gvYS3DgBjN6jo1pB86Kr4k2wojKjVpYRTZ3Us=" -} diff --git a/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel b/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel deleted file mode 100644 index b5dacc58..00000000 --- a/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250519.0-233b207", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250519.0-233b207/source.json b/third_party/modules/rules_zig/20250519.0-233b207/source.json deleted file mode 100644 index 6082ccd8..00000000 --- a/third_party/modules/rules_zig/20250519.0-233b207/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-233b207499826715a3bf1a6eeea6e3052ed49542", - "url": "https://github.com/zml/rules_zig/archive/233b207499826715a3bf1a6eeea6e3052ed49542.tar.gz", - "integrity": "sha256-4gloFoSMPkWCV0Jxikqs1UkfOgQL+NmoXMMnVhtgVvg=" -} diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel b/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel deleted file mode 100644 index 06b1e94e..00000000 --- a/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250530.0-5084f1f", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/source.json b/third_party/modules/rules_zig/20250530.0-5084f1f/source.json deleted file mode 100644 index 081142c0..00000000 --- a/third_party/modules/rules_zig/20250530.0-5084f1f/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-5084f1f3a151dfc697900274c19813a4260217c6", - "url": "https://github.com/zml/rules_zig/archive/5084f1f3a151dfc697900274c19813a4260217c6.tar.gz", - "integrity": "sha256-KpZkIDnKG0sXtRhsE0SDoj3QeeHRKQCRjm6LzxmueRA=" -} diff --git a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel deleted file mode 100644 index 4189634c..00000000 --- a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250613.0-567662a", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250613.0-567662a/source.json b/third_party/modules/rules_zig/20250613.0-567662a/source.json deleted file mode 100644 index 4dbdafee..00000000 --- a/third_party/modules/rules_zig/20250613.0-567662a/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-567662a3ce5e87894950d56c69d8de5ad6e0b5f0", - "url": "https://github.com/zml/rules_zig/archive/567662a3ce5e87894950d56c69d8de5ad6e0b5f0.tar.gz", - "integrity": "sha256-PjFpDJXO0BGx2CAZ54Ppv6d4TwTDOWS+mVangKQFvZc=" -} diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel b/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel deleted file mode 100644 index 69a0dc9b..00000000 --- a/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20250714.0-b14a4f1", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json b/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json deleted file mode 100644 index 5eb58847..00000000 --- a/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-b14a4f1bba1a766311f75513e922cc8c2179e1e3", - "url": "https://github.com/zml/rules_zig/archive/b14a4f1bba1a766311f75513e922cc8c2179e1e3.tar.gz", - "integrity": "sha256-XEdYlvjs54SVoLGM4x9J+uofCZk2UIO1nbsqwV+Blqs=" -} diff --git a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel deleted file mode 100644 index 3c64eb32..00000000 --- a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel +++ /dev/null @@ -1,75 +0,0 @@ -module( - name = "rules_zig", - version = "20250821.0-be53625", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") - -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) - -# Hack to get around a cc_common.link limitation. -# See https://github.com/bazelbuild/bazel/pull/23838 -cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") - -cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250821.0-be53625/source.json b/third_party/modules/rules_zig/20250821.0-be53625/source.json deleted file mode 100644 index 959b2dbc..00000000 --- a/third_party/modules/rules_zig/20250821.0-be53625/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-be53625afb13e73856ee4cab38d5aad9f86f63ef", - "url": "https://github.com/zml/rules_zig/archive/be53625afb13e73856ee4cab38d5aad9f86f63ef.tar.gz", - "integrity": "sha256-IsqEl3BHW/vttyHpVBx5WzO3an/K3eZZx8RJk7nVx8s=" -} diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel deleted file mode 100644 index 7620c53c..00000000 --- a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel +++ /dev/null @@ -1,75 +0,0 @@ -module( - name = "rules_zig", - version = "20250827.0-35b6d57", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") - -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) - -# Hack to get around a cc_common.link limitation. -# See https://github.com/bazelbuild/bazel/pull/23838 -cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") - -cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json deleted file mode 100644 index e42d62ba..00000000 --- a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "strip_prefix": "rules_zig-35b6d57e94f3eb08e978c1133314606f4f38e216", - "url": "https://github.com/zml/rules_zig/archive/35b6d57e94f3eb08e978c1133314606f4f38e216.tar.gz", - "integrity": "sha256-FDnAqynTD2LB3W/IaBgocmnLz8CA9nyHZYYntC4plUU=" -} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json deleted file mode 100644 index e7937f1e..00000000 --- a/third_party/modules/rules_zig/metadata.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "homepage": "https://github.com/zml/rules_zig", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": ["github:zml/rules_zig"], - "versions": [ - "20240904.0-010da15", - "20240909.0-37f17ff", - "20240912.0-41bfe84", - "20240913.0-1957d05", - "20250314.0-b9739c6", - "20250519.0-233b207", - "20250613.0-567662a", - "20250714.0-b14a4f1", - "20250821.0-be53625", - "20250827.0-35b6d57" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel deleted file mode 100644 index af94d93f..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "sentencepiece", - version = "20240618.0-d7ace0a", - compatibility_level = 1, -) - -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel deleted file mode 100644 index 43639e89..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel +++ /dev/null @@ -1,7 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") - -proto_library( - name = "sentencepiece_model_proto", - srcs = ["src/sentencepiece_model.proto"], - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json b/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json deleted file mode 100644 index 588c9bcb..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "sentencepiece-d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8", - "url": "https://github.com/google/sentencepiece/archive/d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8.tar.gz", - "integrity": "sha256-bt8QFNNIpqcJoCNMUrmWfUf0uNobtk8pW+XbHL+ZRLg=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/sentencepiece/metadata.json b/third_party/modules/sentencepiece/metadata.json deleted file mode 100644 index 2c9758c9..00000000 --- a/third_party/modules/sentencepiece/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/google/sentencepiece", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "google/sentencepiece" - ], - "versions": [ - "20240618.0-d7ace0a" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel deleted file mode 100644 index 29e45e04..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240829.0-54aa1a5", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240823.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch deleted file mode 100644 index 82ca0c4d..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch +++ /dev/null @@ -1,83 +0,0 @@ -From f11d9bd7639f63cba681caa39cea27af114a4a71 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 2 Sep 2024 23:02:28 +0200 -Subject: [PATCH 1/2] Remove duplicated symbols in StablehloApi.h - -This causes C compilers to choke. - -Refs #2494 ---- - stablehlo/integrations/c/StablehloApi.cpp | 2 +- - stablehlo/integrations/c/StablehloApi.h | 15 +-------------- - stablehlo/integrations/python/StablehloApi.cpp | 2 +- - 3 files changed, 3 insertions(+), 16 deletions(-) - -diff --git a/stablehlo/integrations/c/StablehloApi.cpp b/stablehlo/integrations/c/StablehloApi.cpp -index 8d922198..9b41ff21 100644 ---- a/stablehlo/integrations/c/StablehloApi.cpp -+++ b/stablehlo/integrations/c/StablehloApi.cpp -@@ -98,7 +98,7 @@ MlirLogicalResult stablehloSerializePortableArtifact( - return mlirLogicalResultSuccess(); - } - --MlirLogicalResult stablehloDeserializePortableArtifact( -+MlirLogicalResult stablehloDeserializePortableArtifactToBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void *userData) { - mlir::detail::CallbackOstream stream(callback, userData); - if (failed(mlir::stablehlo::deserializePortableArtifact(unwrap(artifactStr), -diff --git a/stablehlo/integrations/c/StablehloApi.h b/stablehlo/integrations/c/StablehloApi.h -index 4c542508..f94dca8d 100644 ---- a/stablehlo/integrations/c/StablehloApi.h -+++ b/stablehlo/integrations/c/StablehloApi.h -@@ -76,16 +76,6 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - MlirStringRef moduleStr, MlirStringRef targetVersion, - MlirStringCallback callback, void* userData); - --// Write a StableHLO program expressed as a string (either prettyprinted MLIR --// module or MLIR bytecode) to a portable artifact. --// Can fail if `moduleStr` cannot be parsed, or if it cannot be expressed in the --// `targetVersion` version of StableHLO, e.g. if it's using new or removed --// features, or if it involves unsupported dialects. --// Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( -- MlirModule moduleStr, MlirStringRef targetVersion, -- MlirStringCallback callback, void* userData); -- - // Read a StableHLO program from a portable artifact, returning the module as - // MLIR bytecode. Note, this bytecode returned is not a portable artifact, - // and has the stability of returning textual assembly format. Bytecode is -@@ -93,7 +83,7 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - // Can fail if `artifactStr` cannot be expressed in the current version of - // StableHLO, e.g. if it's using incompatible features. - // Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifact( -+MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifactAsBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void* userData); - - // Read a StableHLO program from a portable artifact, returning the module as -@@ -109,9 +99,6 @@ MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( - - // Call the Interpreter, returns MlirArrayAttr of dense element - // MlirAttribute results --MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( -- MlirStringRef artifactStr, MlirContext ctx); -- - // Entrypoint for calling the StableHLO reference interpreter. - // Returns an array attribute of dense element attributes for results. - // Sets error code to non-zero on failure. -diff --git a/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/integrations/python/StablehloApi.cpp -index 46a640e1..4229ef76 100644 ---- a/stablehlo/integrations/python/StablehloApi.cpp -+++ b/stablehlo/integrations/python/StablehloApi.cpp -@@ -213,7 +213,7 @@ void AddPortableApi(py::module &m) { - "deserialize_portable_artifact_str", - [](std::string_view artifact) -> py::bytes { - StringWriterHelper accumulator; -- if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifact( -+ if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifactToBytecode( - toMlirStringRef(artifact), accumulator.getMlirStringCallback(), - accumulator.getUserData()))) { - PyErr_SetString(PyExc_ValueError, "failed to deserialize module"); --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch deleted file mode 100644 index 5240a14c..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch +++ /dev/null @@ -1,401 +0,0 @@ -diff --ruN a/stablehlo/BUILD.bazel b/stablehlo/BUILD.bazel ---- stablehlo/BUILD.bazel -+++ stablehlo/BUILD.bazel -@@ -340,6 +340,21 @@ - ], - ) - -+gentbl_cc_library( -+ name = "stablehlo_create_compatibility_expander_inc_gen", -+ tbl_outs = [ -+ ( -+ ["--gen-rewriters"], -+ "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc", -+ ), -+ ], -+ tblgen = "@llvm-project//mlir:mlir-tblgen", -+ td_file = "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td", -+ deps = [ -+ ":stablehlo_ops_td_files", -+ ], -+) -+ - cc_library( - name = "interpreter_ops", - srcs = [ -@@ -1086,6 +1101,7 @@ - "stablehlo/transforms/StablehloAggressiveSimplification.cpp", - "stablehlo/transforms/StablehloCanonicalizeDynamism.cpp", - "stablehlo/transforms/StablehloConvertToSignless.cpp", -+ "stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp", - "stablehlo/transforms/StablehloLegalizeCompositeToCall.cpp", - "stablehlo/transforms/StablehloLegalizeDeprecatedOps.cpp", - "stablehlo/transforms/StablehloLegalizeQDQToQuantizedOp.cpp", -@@ -1109,6 +1125,7 @@ - ":chlo_ops", - ":chlo_rewriters_inc_gen", - ":linalg_passes", -+ ":stablehlo_create_compatibility_expander_inc_gen", - ":stablehlo_legalize_deprecated_ops_inc_gen", - ":stablehlo_ops", - ":stablehlo_ops_inc_gen", -diff --ruN a/stablehlo/stablehlo/dialect/Version.cpp b/stablehlo/stablehlo/dialect/Version.cpp ---- stablehlo/stablehlo/dialect/Version.cpp -+++ stablehlo/stablehlo/dialect/Version.cpp -@@ -82,7 +82,7 @@ - case CompatibilityRequirement::WEEK_4: - return Version(1, 3, 0); // v1.3.0 - Jul 15, 2024 - case CompatibilityRequirement::WEEK_12: -- return Version(1, 0, 0); // v1.0.0 - May 14, 2024 -+ return Version(1, 1, 0); // v1.1.0 - May 30, 2024 - case CompatibilityRequirement::MAX: - return Version::getMinimumVersion(); - } -diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir ---- stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -+++ stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -@@ -0,0 +1,43 @@ -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file -allow-unregistered-dialect --stablehlo-create-compatibility-expander='target=1.0.0' | FileCheck %s --check-prefixes=CHECK -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file --stablehlo-create-compatibility-expander='target=1.6.0' | FileCheck %s --check-prefixes=CHECK-NO-DOWNGRADE -+ -+// ----- -+ -+// CHECK-LABEL @tan_op_non_complex -+// CHECK: %[[sine0:.*]] = stablehlo.sine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[cosine1:.*]] = stablehlo.cosine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[div2:.*]] = stablehlo.divide %[[sine0]], %[[cosine1]] : tensor<4xf64> -+// CHECK-NEXT: return %[[div2]] : tensor<4xf64> -+func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %arg0 : tensor<4xf64> -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+} -+ -+// ----- -+ -+// CHECK-LABEL: @tan_op_complex -+// CHECK: %[[cst:.*]] = stablehlo.constant dense<1.000000e+00> : tensor<4xf64> -+// CHECK: %[[complex:.*]] = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+// CHECK: %[[real:.*]] = stablehlo.real %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[sine:.*]] = stablehlo.sine %[[real]] : tensor<4xf64> -+// CHECK: %[[cosine:.*]] = stablehlo.cosine %[[real]] : tensor<4xf64> -+// CHECK: %[[divide1:.*]] = stablehlo.divide %[[sine]], %[[cosine]] : tensor<4xf64> -+// CHECK: %[[imag:.*]] = stablehlo.imag %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[tanh:.*]] = stablehlo.tanh %[[imag]] : tensor<4xf64> -+// CHECK: %[[complex2:.*]] = stablehlo.complex %[[divide1]], %[[tanh]] : tensor<4xcomplex> -+// CHECK: %[[multiply:.*]] = stablehlo.multiply %[[divide1]], %[[tanh]] : tensor<4xf64> -+// CHECK: %[[negate:.*]] = stablehlo.negate %[[multiply]] : tensor<4xf64> -+// CHECK: %[[complex3:.*]] = stablehlo.complex %[[cst]], %[[negate]] : tensor<4xcomplex> -+// CHECK: %[[divide2:.*]] = stablehlo.divide %[[complex2]], %[[complex3]] : tensor<4xcomplex> -+// CHECK: %[[real2:.*]] = stablehlo.real %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[imag2:.*]] = stablehlo.imag %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: return %[[real2]], %[[imag2]] : tensor<4xf64>, tensor<4xf64> -+func.func @tan_op_complex(%arg0: tensor<4xf64>, %arg1: tensor<4xf64>) -> (tensor<4xf64>, tensor<4xf64>) { -+ %0 = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %0 : tensor<4xcomplex> -+ %1 = stablehlo.tan %0 : tensor<4xcomplex> -+ %2 = stablehlo.real %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ %3 = stablehlo.imag %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ func.return %2, %3 : tensor<4xf64>, tensor<4xf64> -+} -diff --ruN a/stablehlo/stablehlo/transforms/CMakeLists.txt b/stablehlo/stablehlo/transforms/CMakeLists.txt ---- stablehlo/stablehlo/transforms/CMakeLists.txt -+++ stablehlo/stablehlo/transforms/CMakeLists.txt -@@ -20,6 +20,10 @@ - mlir_tablegen(ChloDecompositionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(ChloDecompositionPatternsIncGen) - -+set(LLVM_TARGET_DEFINITIONS StablehloCreateCompatibilityExpanderPatterns.td) -+mlir_tablegen(StablehloCreateCompatibilityExpanderPatterns.h.inc --gen-rewriters) -+add_public_tablegen_target(StablehloCreateCompatibilityExpanderPatternsIncGen) -+ - set(LLVM_TARGET_DEFINITIONS StablehloLegalizeDeprecatedOpsPatterns.td) - mlir_tablegen(StablehloLegalizeDeprecatedOpsPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(StablehloLegalizeDeprecatedOpsPatternsIncGen) -@@ -27,6 +31,7 @@ - set(LLVM_TARGET_DEFINITIONS VhloToVersionPatterns.td) - mlir_tablegen(VhloToVersionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(VhloToVersionPatterns) -+ - - add_mlir_dialect_library(StablehloPasses - PARTIAL_SOURCES_INTENDED -@@ -37,6 +42,7 @@ - StablehloAggressiveSimplification.cpp - StablehloCanonicalizeDynamism.cpp - StablehloConvertToSignless.cpp -+ StablehloCreateCompatibilityExpander.cpp - StablehloLegalizeCompositeToCall.cpp - StablehloLegalizeDeprecatedOps.cpp - StablehloLegalizeQuantToMath.cpp -@@ -53,6 +59,7 @@ - StablehloLegalizeDeprecatedOpsPatternsIncGen - PassesIncGen - VhloToVersionPatterns -+ StablehloCreateCompatibilityExpanderPatternsIncGen - - LINK_LIBS PUBLIC - ChloOps -diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h ---- stablehlo/stablehlo/transforms/Passes.h -+++ stablehlo/stablehlo/transforms/Passes.h -@@ -25,6 +25,7 @@ - #include "mlir/Pass/Pass.h" - #include "mlir/Support/LogicalResult.h" - #include "mlir/Transforms/DialectConversion.h" -+#include "stablehlo/dialect/Version.h" - - namespace mlir { - namespace stablehlo { -@@ -96,6 +97,12 @@ - void populateShapeToStablehloPatterns(MLIRContext *context, - RewritePatternSet *patterns); - -+/// Collection of patterns to create compatibility expander for StableHLO -+/// operations. -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion); -+ - //// Additional pass constructors //// - - std::unique_ptr> createStablehloRefineArgumentsPass( -diff --ruN a/stablehlo/stablehlo/transforms/Passes.td b/stablehlo/stablehlo/transforms/Passes.td ---- stablehlo/stablehlo/transforms/Passes.td -+++ stablehlo/stablehlo/transforms/Passes.td -@@ -292,3 +292,51 @@ - "mlir::stablehlo::StablehloDialect", - ]; - } -+ -+def StablehloCreateCompatibilityExpanderPass : Pass<"stablehlo-create-compatibility-expander", "mlir::func::FuncOp"> { -+ let summary = "Create compatibility expander for StableHLO operations."; -+ -+ let description = [{ -+ StableHLO ops gets updates or new op is introduced in the latest versions. -+ This opt-in pass expands backward compatibility with older StableHLO -+ versions by decomposing newer StableHLO operations into equivalent -+ operations supported by those older versions. -+ -+ Why is this an opt-in pass? -+ -+ Occasionally, StableHLO op enhancements are used to greatly simplify the -+ handling of certain common patterns in the OpenXLA ecosystem. This -+ includes things like TanOp, which has high framework and compiler support, -+ as well as gather/scatter batching dimensions, which can be represented -+ using slices, but makes sharding much more difficult. For this category of -+ new features, we do not offer automatic downgrade, since it may throw away -+ important information used in subsequent optimizations. This pass can be -+ used to expand these ops based on a target version to maximize compatibility -+ at the expense of potentially less optimal compilation. -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+ } -+ ``` -+ -+ will become: -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %0 = stablehlo.sine %arg0 : tensor<4xf64> -+ %1 = stablehlo.cosine %arg0 : tensor<4xf64> -+ %2 = stablehlo.divide %0, %1 : tensor<4xf64> -+ return %2 : tensor<4xf64> -+ } -+ ``` -+ }]; -+ let options = [ -+ Option<"targetVersionOption", "target", "std::string", "", -+ "The target version. Must be a version of the form #.#.#.">, -+ ]; -+ let dependentDialects = [ -+ "mlir::stablehlo::StablehloDialect", -+ ]; -+} -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -@@ -0,0 +1,128 @@ -+/* Copyright 2024 The StableHLO Authors. All Rights Reserved. -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ http://www.apache.org/licenses/LICENSE-2.0 -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+ -+#include -+ -+#include "llvm/ADT/APFloat.h" -+#include "llvm/Support/ErrorHandling.h" -+#include "mlir/Dialect/Func/IR/FuncOps.h" -+#include "mlir/IR/BuiltinAttributes.h" -+#include "mlir/IR/PatternMatch.h" -+#include "mlir/Support/LLVM.h" -+#include "mlir/Transforms/DialectConversion.h" -+#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -+#include "stablehlo/dialect/StablehloOps.h" -+#include "stablehlo/dialect/Version.h" -+#include "stablehlo/transforms/Passes.h" -+ -+namespace mlir { -+namespace stablehlo { -+#define GEN_PASS_DEF_STABLEHLOCREATECOMPATIBILITYEXPANDERPASS -+#include "stablehlo/transforms/Passes.h.inc" -+ -+namespace { -+ -+//===----------------------------------------------------------------------===// -+// Helpers. -+//===----------------------------------------------------------------------===// -+ -+// Creates a constant with all ones. -+static Value createConstantWithAllOnes(OpBuilder &b, Location loc, Value val) { -+ auto shapedTy = dyn_cast(val.getType()); -+ if (!shapedTy) llvm_unreachable("Unsupported shaped type."); -+ -+ mlir::DenseElementsAttr elementsAttr = -+ mlir::DenseElementsAttr::get(shapedTy, 1.0); -+ -+ return b.create(loc, val.getType(), -+ elementsAttr); -+} -+ -+// Check user-specified target version. -+vhlo::Version validateTargetVersion(llvm::StringRef versionRef) { -+ auto failOrVersion = vhlo::Version::fromString(versionRef); -+ if (failed(failOrVersion)) { -+ assert(!versionRef.empty() && -+ "No target version specified. Target version must be of the form " -+ "`#.#.#`."); -+ assert(versionRef.empty() && -+ "Invalid target version argument. Target version must be of the " -+ "form `#.#.#`."); -+ } -+ vhlo::Version targetVersion = *failOrVersion; -+ assert((vhlo::Version::getMinimumVersion() <= targetVersion) && -+ "target version is less than minimum supported."); -+ assert((targetVersion <= vhlo::Version::getCurrentVersion()) && -+ "target version is greater than current version."); -+ return targetVersion; -+} -+ -+//===----------------------------------------------------------------------===// -+// Pass -+//===----------------------------------------------------------------------===// -+ -+struct StablehloCreateCompatibilityExpanderPass -+ : public impl::StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass> { -+ StablehloCreateCompatibilityExpanderPass() -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>() {} -+ StablehloCreateCompatibilityExpanderPass( -+ const StablehloCreateCompatibilityExpanderPassOptions &opts) -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>(opts) {} -+ -+ public: -+ LogicalResult initialize(MLIRContext *context) override { -+ auto targetVersion = validateTargetVersion(targetVersionOption); -+ -+ config.useTopDownTraversal = true; -+ RewritePatternSet patterns_(context); -+ populateStablehloCreateCompatibilityExpanderPatterns(&patterns_, context, -+ targetVersion); -+ patterns = std::move(patterns_); -+ return success(); -+ } -+ -+ void runOnOperation() override { -+ auto func = getOperation(); -+ if (failed(applyPatternsAndFoldGreedily(func, patterns, config))) { -+ func.emitError( -+ "Failed to converge StableHLOCreateCompatibilityExpanderPass in ") -+ << config.maxIterations << " iterations"; -+ signalPassFailure(); -+ } -+ } -+ -+ private: -+ FrozenRewritePatternSet patterns; -+ GreedyRewriteConfig config; -+}; -+ -+#include "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc" -+ -+} // namespace -+ -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion) { -+ // StableHLO TanOp is introduced in v1.4.0. -+ if (targetVersion < vhlo::Version(1, 4, 0)) { -+ patterns->add(context); -+ patterns->add(context); -+ } -+} -+ -+} // namespace stablehlo -+} // namespace mlir -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -@@ -0,0 +1,47 @@ -+/* Copyright 2022 The StableHLO Authors. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+include "mlir/IR/OpBase.td" -+include "stablehlo/dialect/StablehloOps.td" -+ -+def ComplexElementType : Type< -+ CPred<"isa(cast($_self).getElementType())">, -+ "Complex element type">; -+ -+def NonComplexElementType : Type< -+ CPred<"!isa(cast($_self).getElementType())">, -+ "Non-complex element type">; -+ -+def createConstantWithAllOnes : NativeCodeCall<"createConstantWithAllOnes($_builder, $_loc, $0)">; -+ -+// Express `tan` as -+// sine(x) / cosine(x) -+def TanOp_CompatiblityExpander : Pat<(StableHLO_TanOp NonComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_SineOp $input), -+ (StableHLO_CosineOp $input) -+ )>; -+ -+// Express `tan(a + bi)` as -+// (tan(a) + i tanh(b)) / (1 - i tan(a) * tanh(b)) -+def TanOp_ComplexElementType_CompatiblityExpander : Pat<(StableHLO_TanOp ComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_ComplexOp -+ (StableHLO_TanOp:$tan (StableHLO_RealOp $input)), -+ (StableHLO_TanhOp:$tanh (StableHLO_ImagOp $input))), -+ (StableHLO_ComplexOp -+ (createConstantWithAllOnes $tan), -+ (StableHLO_NegOp (StableHLO_MulOp $tan, $tanh))) -+ )>; - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json b/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json deleted file mode 100644 index 6d271a53..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "strip_prefix": "stablehlo-54aa1a57178251981da616b877dda1a88d840d11", - "url": "https://github.com/openxla/stablehlo/archive/54aa1a57178251981da616b877dda1a88d840d11.tar.gz", - "integrity": "sha256-PFXXs3vQ+WlMiCC8twbGEFbsBEF/67Z4qtmq8CtUSUU=", - "overlay": { - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Remove-duplicated-symbols-in-StablehloApi.h.patch": "" - } -} diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel deleted file mode 100644 index 7cbfb936..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240917.0-78c753a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240919.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/source.json b/third_party/modules/stablehlo/20240917.0-78c753a/source.json deleted file mode 100644 index cd677bd8..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-78c753ad13ad8205cacc5fcc12418c1ac97276c7", - "url": "https://github.com/openxla/stablehlo/archive/78c753ad13ad8205cacc5fcc12418c1ac97276c7.tar.gz", - "integrity": "sha256-2qFcPLXs7glf7RFDUiqbKXiBBamW7n8a30egR4Yu/bc=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel deleted file mode 100644 index 637e0e76..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20241021.0-1c0b606", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20241022.0-6c4267f") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/source.json b/third_party/modules/stablehlo/20241021.0-1c0b606/source.json deleted file mode 100644 index e67596eb..00000000 --- a/third_party/modules/stablehlo/20241021.0-1c0b606/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-f7f8e4e35296deeff2e12e39421ac8d9599ba340", - "url": "https://github.com/openxla/stablehlo/archive/f7f8e4e35296deeff2e12e39421ac8d9599ba340.tar.gz", - "integrity": "sha256-/chY6BYV7KcrXIuqXTNIUknISCiA1JYhWE8gG5cEeV4=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel deleted file mode 100644 index a045302f..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20241220.0-38bb2f9", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250102.0-f739aa4") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json b/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json deleted file mode 100644 index 9370183f..00000000 --- a/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-38bb2f9bf63b714e8a49fe34a478139058ee1660", - "url": "https://github.com/openxla/stablehlo/archive/38bb2f9bf63b714e8a49fe34a478139058ee1660.tar.gz", - "integrity": "sha256-TuaG8q6WCdJGl4ECpu8YO8ORYcRo0NSff3xHKVj8Ixw=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel deleted file mode 100644 index 2aabd323..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250117.0-c125b32", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250117.0-bf17016") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/source.json b/third_party/modules/stablehlo/20250117.0-c125b32/source.json deleted file mode 100644 index fd8426be..00000000 --- a/third_party/modules/stablehlo/20250117.0-c125b32/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-c125b3284819fec57120231cf4430657dab7b881", - "url": "https://github.com/openxla/stablehlo/archive/c125b3284819fec57120231cf4430657dab7b881.tar.gz", - "integrity": "sha256-h4iBDLVbK2JZRDY7qdmNdns25AoVjpLekKdmsETEDf8=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel deleted file mode 100644 index 7e9accd0..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250201.0-7775e3e", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250201.0-386af4a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/source.json b/third_party/modules/stablehlo/20250201.0-7775e3e/source.json deleted file mode 100644 index e3e17ea2..00000000 --- a/third_party/modules/stablehlo/20250201.0-7775e3e/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-7775e3e2f98fe458f227cdc41aec567535d1c796", - "url": "https://github.com/openxla/stablehlo/archive/7775e3e2f98fe458f227cdc41aec567535d1c796.tar.gz", - "integrity": "sha256-HM/MIs3FeN4PxcYgFsjj04tVJ4u4usGiYGSa9BwT5bg=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel deleted file mode 100644 index 8c462655..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20250217.0-4598975", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20250217.0-0e779ad") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250217.0-4598975/source.json b/third_party/modules/stablehlo/20250217.0-4598975/source.json deleted file mode 100644 index 6988aba7..00000000 --- a/third_party/modules/stablehlo/20250217.0-4598975/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-459897561d365ef97caba46984847f9184d472ec", - "url": "https://github.com/openxla/stablehlo/archive/459897561d365ef97caba46984847f9184d472ec.tar.gz", - "integrity": "sha256-J2KXzmX3uAJiwuMLl06n8dyb9l1TCzGZ62IULyAUcec=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/metadata.json b/third_party/modules/stablehlo/metadata.json deleted file mode 100644 index d0e5fef6..00000000 --- a/third_party/modules/stablehlo/metadata.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "homepage": "https://github.com/openxla/stablehlo", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/stablehlo" - ], - "versions": [ - "20240829.0-54aa1a5", - "20240917.0-78c753a", - "20241021.0-1c0b606", - "20241220.0-38bb2f9", - "20250117.0-c125b32", - "20250201.0-7775e3e", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel deleted file mode 100644 index 739fe835..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240902.0-d18cd64", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240829.0-54aa1a5") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240902.0-d18cd64/source.json b/third_party/modules/xla/20240902.0-d18cd64/source.json deleted file mode 100644 index 359f9b12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d18cd64b7cd61a2ade10089665ac104f639101b1", - "url": "https://github.com/openxla/xla/archive/d18cd64b7cd61a2ade10089665ac104f639101b1.tar.gz", - "integrity": "sha256-EtKhjU91STBceUmg0TUE6cPeRkeSz3LI2S1i3EFMj/E=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel deleted file mode 100644 index bd3a0046..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240919.0-1b18dd6", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240917.0-78c753a") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240919.0-1b18dd6/source.json b/third_party/modules/xla/20240919.0-1b18dd6/source.json deleted file mode 100644 index 9853728a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d391119197eab771a84c1f8a59a7f50b7da4b43d", - "url": "https://github.com/openxla/xla/archive/d391119197eab771a84c1f8a59a7f50b7da4b43d.tar.gz", - "integrity": "sha256-ToPcvPhoGX6Ny3yLx6kJlQSwj7k5Xy4T+BjIp3DAj/s=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel deleted file mode 100644 index 22019c9e..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20241025.0-4663f04", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20241021.0-1c0b606") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20241025.0-4663f04/source.json b/third_party/modules/xla/20241025.0-4663f04/source.json deleted file mode 100644 index 33327387..00000000 --- a/third_party/modules/xla/20241025.0-4663f04/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-4663f04d2653d686833f9c306bd0d899c3127358", - "url": "https://github.com/openxla/xla/archive/4663f04d2653d686833f9c306bd0d899c3127358.tar.gz", - "integrity": "sha256-tQRn0TXkPcmKpCcIuTYM10XNB1qC/d6++4rA1mlnnKw=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel deleted file mode 100644 index 5261911d..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250103.0-5f1fe6a", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20241220.0-38bb2f9") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/source.json b/third_party/modules/xla/20250103.0-5f1fe6a/source.json deleted file mode 100644 index 690bbb4b..00000000 --- a/third_party/modules/xla/20250103.0-5f1fe6a/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-5f1fe6a2f35756238e4a0a0c4e754d6454296efe", - "url": "https://github.com/openxla/xla/archive/5f1fe6a2f35756238e4a0a0c4e754d6454296efe.tar.gz", - "integrity": "sha256-0+kdYIla4F63oPfACtw6LmmMLdKTFlBBrgvc9PmxFoI=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel deleted file mode 100644 index fd99777d..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250122.0-cc075be", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250122.0-cc075be/source.json b/third_party/modules/xla/20250122.0-cc075be/source.json deleted file mode 100644 index 4946d805..00000000 --- a/third_party/modules/xla/20250122.0-cc075be/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", - "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", - "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel deleted file mode 100644 index 6a94e649..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250123.0-cc075be", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index 221fa393..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20250123.0-cc075be/source.json b/third_party/modules/xla/20250123.0-cc075be/source.json deleted file mode 100644 index 02869dd7..00000000 --- a/third_party/modules/xla/20250123.0-cc075be/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", - "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", - "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250204-0-6789523/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/MODULE.bazel deleted file mode 100644 index 7d2ebfb7..00000000 --- a/third_party/modules/xla/20250204-0-6789523/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250204-0-6789523", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250201.0-7775e3e") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index a29cbd9e..00000000 --- a/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 4b6a65ce..00000000 --- a/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/source.json b/third_party/modules/xla/20250204-0-6789523/source.json deleted file mode 100644 index ab3abf73..00000000 --- a/third_party/modules/xla/20250204-0-6789523/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", - "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", - "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250204.1-6789523/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/MODULE.bazel deleted file mode 100644 index f0ad4f19..00000000 --- a/third_party/modules/xla/20250204.1-6789523/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20250204.1-6789523", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20250217.0-4598975") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch deleted file mode 100644 index a29cbd9e..00000000 --- a/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --6.5.0 -+8.0.1 -\ No newline at end of file -diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/tsl/third_party/gpus/cuda_configure.bzl -+++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 4b6a65ce..00000000 --- a/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/source.json b/third_party/modules/xla/20250204.1-6789523/source.json deleted file mode 100644 index ab3abf73..00000000 --- a/third_party/modules/xla/20250204.1-6789523/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", - "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", - "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.0.1.patch": "", - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel deleted file mode 100644 index 19a58980..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.0-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") -llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.0-71c67e2/source.json b/third_party/modules/xla/20250317.0-71c67e2/source.json deleted file mode 100644 index c239d180..00000000 --- a/third_party/modules/xla/20250317.0-71c67e2/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel deleted file mode 100644 index 516cef14..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.1-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") -llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 58e3874d..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Wed, 5 Feb 2025 19:25:03 +0100 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 ++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- - 3 files changed, 54 insertions(+), 2 deletions(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index ad2ed95bce..0e7c35c30f 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,7 +69,12 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index a33bd4aa9c..3309194538 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - size_t struct_size; - PJRT_Extension_Type type; - PJRT_Extension_Base* next; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 0375b39d0b..3527a0756e 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" -+#include "xla/ffi/api/c_api.h" -+#include "xla/ffi/api/ffi.h" - #include "xla/ffi/execution_context.h" --#include "xla/ffi/type_id_registry.h" -+ #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, -@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - /*next=*/next, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.43.0 diff --git a/third_party/modules/xla/20250317.1-71c67e2/source.json b/third_party/modules/xla/20250317.1-71c67e2/source.json deleted file mode 100644 index 5577d247..00000000 --- a/third_party/modules/xla/20250317.1-71c67e2/source.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } -} diff --git a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel deleted file mode 100644 index 042d899c..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel +++ /dev/null @@ -1,37 +0,0 @@ -module( - name = "xla", - version = "20250317.2-71c67e2", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl", "python_version_repo") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "llvm-raw", - "stablehlo", -) - -llvm_configure = use_extension("//:llvm.bzl", "llvm_configure") -use_repo(llvm_configure, "llvm-project") diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl deleted file mode 100644 index 57e77361..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_configure_impl(mctx): - _llvm_configure(name = "llvm-project") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm_configure = module_extension( - implementation = _llvm_configure_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl deleted file mode 100644 index dc25c9d6..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") -load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") - -def _tsl_impl(mctx): - python_init_repositories( - requirements = { - "3.11": "//:requirements_lock_3_11.txt", - }, - ) - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["tsl"], - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl deleted file mode 100644 index 192e06f8..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl +++ /dev/null @@ -1,60 +0,0 @@ -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - pybind11_bazel() - triton() - llvm("llvm-raw") - stablehlo() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 58e3874d..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,131 +0,0 @@ -From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Wed, 5 Feb 2025 19:25:03 +0100 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 ++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- - 3 files changed, 54 insertions(+), 2 deletions(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index ad2ed95bce..0e7c35c30f 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,7 +69,12 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index a33bd4aa9c..3309194538 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - size_t struct_size; - PJRT_Extension_Type type; - PJRT_Extension_Base* next; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 0375b39d0b..3527a0756e 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" -+#include "xla/ffi/api/c_api.h" -+#include "xla/ffi/api/ffi.h" - #include "xla/ffi/execution_context.h" --#include "xla/ffi/type_id_registry.h" -+ #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, -@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - /*next=*/next, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.43.0 diff --git a/third_party/modules/xla/20250317.2-71c67e2/source.json b/third_party/modules/xla/20250317.2-71c67e2/source.json deleted file mode 100644 index 9d579abc..00000000 --- a/third_party/modules/xla/20250317.2-71c67e2/source.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", - "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", - "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "tsl.bzl": "", - "workspace.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" - } -} diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel deleted file mode 100644 index 74c9eaad..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250527.0-cb67f2f", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl deleted file mode 100644 index b234d122..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl +++ /dev/null @@ -1,71 +0,0 @@ -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party:repo.bzl", "tf_vendored") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch deleted file mode 100644 index 4173db86..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch +++ /dev/null @@ -1,186 +0,0 @@ -From 2a62bc5df9774810313142eb0c9390aab3cd18f8 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Thu, 29 May 2025 08:02:32 +0200 -Subject: [PATCH] Revert "Add optional allowOtherDialects field to - stablehlo.serialize_portable_artifact." - -Commit: https://github.com/openxla/xla/commit/e7137a383809a24875a95237b1d1f6485acdf710 - -Issue: C does not support default arguments with ZigTranslateC ---- - third_party/stablehlo/temporary.patch | 151 ++++---------------------- - 1 file changed, 21 insertions(+), 130 deletions(-) - -diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch -index 6e1fd159f9..d17c141b18 100755 ---- a/third_party/stablehlo/temporary.patch -+++ b/third_party/stablehlo/temporary.patch -@@ -1,3 +1,23 @@ -+diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.cpp b/stablehlo/stablehlo/dialect/StablehloOps.cpp -+--- stablehlo/stablehlo/dialect/StablehloOps.cpp -++++ stablehlo/stablehlo/dialect/StablehloOps.cpp -+@@ -511,12 +511,10 @@ -+ void CustomCallOp::getEffects( -+ SmallVectorImpl>& -+ effects) { -+- // Note: `has_side_effect` "defaults" to `false` but isn't required to exist. -+- // This semantic contradiction means, in practical terms, that the attribute -+- // won't exist by default but should be *treated* as `false` if missing. -+- // `getHasSideEffect()` abstracts this nuance away and returns `false` by -+- // default, whereas `getHasSideEffectAttr()` may return a null attribute. -+- if (!getHasSideEffect()) return; -++ // CustomCall has "all possible effects" unless the has_side_effect is present -++ // and set to false. -++ auto hasSideEffect = getHasSideEffectAttr(); -++ if (hasSideEffect && !hasSideEffect.getValue()) return; -+ effects.emplace_back(MemoryEffects::Allocate::get()); -+ effects.emplace_back(MemoryEffects::Free::get()); -+ effects.emplace_back(MemoryEffects::Write::get()); - diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.h b/stablehlo/stablehlo/dialect/StablehloOps.h - --- stablehlo/stablehlo/dialect/StablehloOps.h - +++ stablehlo/stablehlo/dialect/StablehloOps.h -@@ -82,135 +102,6 @@ diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.td b/stablehlo/stablehlo/d - ]> { - let summary = "Recv operation"; - let description = [{ --diff --ruN a/stablehlo/stablehlo/dialect/TypeInference.cpp b/stablehlo/stablehlo/dialect/TypeInference.cpp ----- stablehlo/stablehlo/dialect/TypeInference.cpp --+++ stablehlo/stablehlo/dialect/TypeInference.cpp --@@ -879,7 +879,8 @@ -- -- auto replicaIds = replicaGroups.getValues(); -- --- llvm::SmallSet replicaIdsSeen; --+ // Large programs can have many replicas, use a set with efficient lookup. --+ llvm::DenseSet replicaIdsSeen; -- for (int64_t replicaId : replicaIds) { -- // Replica groups are stored in a 2D tensor. If the op supports non-uniform -- // groups, null replica IDs are stored as -1. --@@ -1841,6 +1842,7 @@ -- /*allGroupsMustHaveSameSize=*/true, -- /*useGlobalDeviceIds=*/false, splitCount))) -- return failure(); --+ -- for (const Value& operand : operands) { -- auto operandType = cast(operand.getType()); -- --@@ -3562,6 +3564,19 @@ -- DenseIntElementsAttr replicaGroups, -- int64_t channelId, bool useGlobalDeviceIds, -- ValueRange results) { --+ // all_gather_i3, all_gather_c2, all_gather_c4 --+ if (failed(verifyReplicaGroups(location, replicaGroups, --+ /*allGroupsMustHaveSameSize=*/true, --+ useGlobalDeviceIds, --+ /*expectedGroupSize=*/std::nullopt))) --+ return failure(); --+ --+ // all_gather_c5 --+ if (useGlobalDeviceIds && channelId < 0) --+ return emitOptionalError( --+ location, --+ "channel_id cannot be negative when useGlobalDeviceIds is set"); --+ -- for (const auto& [operand, result] : llvm::zip(operands, results)) { -- auto operandType = cast(operand.getType()); -- auto resultType = cast(result.getType()); --@@ -3576,19 +3591,6 @@ -- return emitOptionalError( -- location, -- "dimension size of operand at 'all_gather_dim' cannot be zero"); --- --- // all_gather_i3, all_gather_c2, all_gather_c4 --- if (failed(verifyReplicaGroups(location, replicaGroups, --- /*allGroupsMustHaveSameSize=*/true, --- useGlobalDeviceIds, --- /*expectedGroupSize=*/std::nullopt))) --- return failure(); --- --- // all_gather_c5 --- if (useGlobalDeviceIds && channelId < 0) --- return emitOptionalError( --- location, --- "channel_id cannot be negative when useGlobalDeviceIds is set"); -- -- // all_gather_c6 -- if (resultType.getRank() != operandType.getRank()) --@@ -3788,7 +3790,7 @@ -- "but instead it is of rank ", replicaGroupType.getRank()); -- -- auto replicaIds = replicaGroups.getValues(); --- llvm::SmallSet replicaIdsSeen; --+ llvm::DenseSet replicaIdsSeen; -- for (int64_t replicaId : replicaIds) { -- // collective_broadcast_c2 -- // We only check that is is not negative, as it is impossible --diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp ----- stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp --+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp --@@ -78,10 +78,11 @@ -- -- MlirLogicalResult stablehloSerializePortableArtifactFromModule( -- MlirModule moduleStr, MlirStringRef targetVersion, --- MlirStringCallback callback, void *userData) { --+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -- mlir::detail::CallbackOstream stream(callback, userData); -- if (failed(mlir::stablehlo::serializePortableArtifact( --- unwrap(moduleStr), unwrap(targetVersion), stream))) --+ unwrap(moduleStr), unwrap(targetVersion), stream, --+ allowOtherDialects))) -- return mlirLogicalResultFailure(); -- return mlirLogicalResultSuccess(); -- } --diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h ----- stablehlo/stablehlo/integrations/c/StablehloDialectApi.h --+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.h --@@ -92,7 +92,8 @@ -- stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -- MlirStringRef targetVersion, -- MlirStringCallback callback, --- void* userData); --+ void* userData, --+ bool allowOtherDialects = false); -- -- // Read a StableHLO program from a portable artifact, returning the module as -- // MLIR bytecode. Note, this bytecode returned is not a portable artifact, --diff --ruN a/stablehlo/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/stablehlo/integrations/python/StablehloApi.cpp ----- stablehlo/stablehlo/integrations/python/StablehloApi.cpp --+++ stablehlo/stablehlo/integrations/python/StablehloApi.cpp --@@ -102,20 +102,22 @@ -- // -- m.def( -- "serialize_portable_artifact", --- [](MlirModule module, std::string_view target) -> nb::bytes { --+ [](MlirModule module, std::string_view target, --+ bool allowOtherDialects) -> nb::bytes { -- StringWriterHelper accumulator; -- if (mlirLogicalResultIsFailure( -- stablehloSerializePortableArtifactFromModule( -- module, toMlirStringRef(target), -- accumulator.getMlirStringCallback(), --- accumulator.getUserData()))) { --+ accumulator.getUserData(), allowOtherDialects))) { -- throw nb::value_error("failed to serialize module"); -- } -- -- std::string serialized = accumulator.toString(); -- return nb::bytes(serialized.data(), serialized.size()); -- }, --- nb::arg("module"), nb::arg("target")); --+ nb::arg("module"), nb::arg("target"), --+ nb::arg("allow_other_dialects") = false); -- -- m.def( -- "deserialize_portable_artifact", - diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir - --- stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir - +++ stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir -@@ -223,4 +114,4 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless. - %7 = builtin.unrealized_conversion_cast %6 : memref to memref - func.return %7 : memref - } -- -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250527.0-cb67f2f/source.json b/third_party/modules/xla/20250527.0-cb67f2f/source.json deleted file mode 100644 index 86bdd83a..00000000 --- a/third_party/modules/xla/20250527.0-cb67f2f/source.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", - "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", - "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } -} diff --git a/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel deleted file mode 100644 index 46eb7d18..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250612.0-6e48cbb", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel deleted file mode 100644 index 74c9eaad..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel +++ /dev/null @@ -1,52 +0,0 @@ -module( - name = "xla", - version = "20250527.0-cb67f2f", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl deleted file mode 100644 index b234d122..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl +++ /dev/null @@ -1,71 +0,0 @@ -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//third_party/triton:workspace.bzl", triton = "repo") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("//third_party:repo.bzl", "tf_vendored") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "//third_party/grpc:generate_cc_env_fix.patch", - "//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", - "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch deleted file mode 100644 index 27d52df5..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch +++ /dev/null @@ -1,1855 +0,0 @@ -From 4c0819ac9fb9dfc6156ae4de83fb29e987ade780 Mon Sep 17 00:00:00 2001 -From: Corentin Godeau -Date: Mon, 16 Jun 2025 14:27:47 +0000 -Subject: [PATCH] Update Stablehlo patch - ---- - third_party/stablehlo/temporary.patch | 841 +++++++++++++------------- - 1 file changed, 432 insertions(+), 409 deletions(-) - mode change 100755 => 100644 third_party/stablehlo/temporary.patch - -diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch -old mode 100755 -new mode 100644 -index f54e8341ba..6b25e78467 ---- a/third_party/stablehlo/temporary.patch -+++ b/third_party/stablehlo/temporary.patch -@@ -1,7 +1,47 @@ --diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dialect/VhloAttrs.td ----- stablehlo/stablehlo/dialect/VhloAttrs.td --+++ stablehlo/stablehlo/dialect/VhloAttrs.td --@@ -45,14 +45,6 @@ -+From c48aa0650bf27b81c42799f9218dd90779e36e3b Mon Sep 17 00:00:00 2001 -+From: Corentin Godeau -+Date: Mon, 16 Jun 2025 14:08:48 +0000 -+Subject: [PATCH] Remove default argument that is not valid C -+ -+--- -+ stablehlo/dialect/Version.h | 2 +- -+ stablehlo/dialect/VhloAttrs.td | 14 +- -+ stablehlo/dialect/VhloDialect.td | 1 + -+ stablehlo/dialect/VhloTypes.h | 24 +- -+ .../integrations/c/StablehloDialectApi.h | 2 +- -+ .../stablehlo_aggressive_folder.mlir | 37 +- -+ .../stablehlo_aggressive_simplification.mlir | 2 +- -+ .../transforms/stablehlo_refine_shapes.mlir | 10 +- -+ .../stablehlo_legalize_to_vhlo_mixed.mlir | 38 +- -+ .../tests/vhlo/vhlo_attributes_invalid.mlir | 13 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 8 +- -+ stablehlo/transforms/Passes.h | 7 +- -+ .../transforms/StablehloLegalizeToVhlo.cpp | 48 +- -+ .../transforms/StablehloRefineShapes.cpp | 4 + -+ .../transforms/VhloLegalizeToStablehlo.cpp | 100 ++- -+ stablehlo/transforms/VhloToVersion.cpp | 38 +- -+ .../StablehloAggressiveFolder.cpp | 816 ++++++++++-------- -+ ...ablehloAggressiveSimplificationPatterns.td | 6 +- -+ 18 files changed, 696 insertions(+), 474 deletions(-) -+ -+diff --git a/stablehlo/dialect/Version.h b/stablehlo/dialect/Version.h -+index eb35ad52..6aaef985 100644 -+--- a/stablehlo/dialect/Version.h -++++ b/stablehlo/dialect/Version.h -+@@ -38,7 +38,7 @@ class Version { -+ static FailureOr fromString(llvm::StringRef versionRef); -+ -+ /// Return a Version representing the current VHLO dialect version. -+- static Version getCurrentVersion() { return Version(1, 10, 10); } -++ static Version getCurrentVersion() { return Version(1, 11, 0); } -+ -+ /// Return a Version representing the minimum supported VHLO dialect version. -+ static Version getMinimumVersion() { return Version(0, 9, 0); } -+diff --git a/stablehlo/dialect/VhloAttrs.td b/stablehlo/dialect/VhloAttrs.td -+index bf75ad27..ab48630a 100644 -+--- a/stablehlo/dialect/VhloAttrs.td -++++ b/stablehlo/dialect/VhloAttrs.td -+@@ -45,14 +45,6 @@ class VHLO_AttrDef - def VHLO_ArrayAttrV1 : VHLO_AttrDef<"ArrayV1", "0.9.0", "current"> { - let mnemonic = "array_v1"; - let parameters = (ins ArrayRefParameter<"mlir::Attribute">:$value); -@@ -16,7 +56,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial - let assemblyFormat = "`<` custom($value) `>`"; - } - --@@ -75,9 +67,9 @@ -+@@ -75,9 +67,9 @@ def VHLO_DictionaryAttrV1 : VHLO_AttrDef<"DictionaryV1", "0.9.0", "current"> { - LogicalResult DictionaryV1Attr::verify( - llvm::function_ref errFn, - ArrayRef> value) { -@@ -29,36 +69,48 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial - return success(); - } - }]; --diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/dialect/VhloTypes.h ----- stablehlo/stablehlo/dialect/VhloTypes.h --+++ stablehlo/stablehlo/dialect/VhloTypes.h --@@ -27,20 +27,23 @@ -+diff --git a/stablehlo/dialect/VhloDialect.td b/stablehlo/dialect/VhloDialect.td -+index 4e50cb32..edd220e3 100644 -+--- a/stablehlo/dialect/VhloDialect.td -++++ b/stablehlo/dialect/VhloDialect.td -+@@ -49,6 +49,7 @@ def VHLO_Dialect : Dialect { -+ 1.8.0: Introduce `f4E2M1FN`, `f6E2M3FN`, `f6E3M2FN` and `f8E8M0FNU` types. -+ 1.9.0: Add `ResultAccuracy` attribute to `exp` op. -+ 1.10.0: Add `ResultAccuracy` attribute to `cbrt`, `cosine`, `exponential`, `exponential_minus_one`, `log`, `log_plus_one`, `logistic`, `rsqrt`, `sine`, `sqrt`, `tan` and `tanh` ops. -++ 1.11.0: Allow (de)serializing VHLO programs mixed with potentially unstable dialects. -+ }]; -+ -+ let useDefaultAttributePrinterParser = 0; -+diff --git a/stablehlo/dialect/VhloTypes.h b/stablehlo/dialect/VhloTypes.h -+index e5aee254..f3a8d68d 100644 -+--- a/stablehlo/dialect/VhloTypes.h -++++ b/stablehlo/dialect/VhloTypes.h -+@@ -27,20 +27,23 @@ limitations under the License. - namespace mlir { - namespace vhlo { - - -class VhloTypeConverterBase : public TypeConverter { --- public: --- VhloTypeConverterBase() : TypeConverter(){}; --- --- virtual ~VhloTypeConverterBase() = default; --- --- virtual Attribute convertEncoding(Attribute attr) const = 0; ---}; -- -- // This class is used to manage conversions between VHLO and Builtin -- // dialects. ---class VhloTypeConverter : public VhloTypeConverterBase { -++ -++// This class is used to manage conversions between VHLO and Builtin -++// dialects. - +class VhloTypeConverter : public TypeConverter { - public: --- VhloTypeConverter() : VhloTypeConverterBase() {} -+- VhloTypeConverterBase() : TypeConverter(){}; - + VhloTypeConverter() : TypeConverter(), allowOtherDialects(false) {} - + VhloTypeConverter(bool allowOtherDialects) - + : TypeConverter(), allowOtherDialects(allowOtherDialects) {} --+ -+ -+- virtual ~VhloTypeConverterBase() = default; - + virtual ~VhloTypeConverter() = default; --+ --+ virtual Attribute convertEncoding(Attribute attr) const = 0; --+ -+ -+ virtual Attribute convertEncoding(Attribute attr) const = 0; -+-}; -+ -+-// This class is used to manage conversions between VHLO and Builtin -+-// dialects. -+-class VhloTypeConverter : public VhloTypeConverterBase { -+- public: -+- VhloTypeConverter() : VhloTypeConverterBase() {} - + Attribute convertUnknownAttribute(Attribute attr) const { - + if (allowOtherDialects) return attr; - + return {}; -@@ -66,7 +118,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale - - // A subclass can call this method to add conversions from VHLO -> Builtin - // types. Note that conversions are applied in reverse order, with the most --@@ -58,6 +61,9 @@ -+@@ -58,6 +61,9 @@ class VhloTypeConverter : public VhloTypeConverterBase { - - // Mark unrealized casts as legal. Useful for dialect mixing. - void addUnrealizedMaterializations(); -@@ -76,19 +128,34 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale - }; - - // Autogenerated VHLO type printers and parsers. --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -+index 758965d0..c5239671 100644 -+--- a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir -++++ b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir - @@ -1,4 +1,4 @@ - -// RUN: stablehlo-opt --stablehlo-aggressive-folder --split-input-file --verify-diagnostics %s | FileCheck %s - +// RUN: stablehlo-opt --stablehlo-aggressive-folder=fold-op-element-limit=100 --split-input-file --verify-diagnostics %s | FileCheck %s - - //////// - // AddOp --@@ -42,6 +42,21 @@ -+@@ -41,6 +41,21 @@ func.func @broadcast_in_dim_fold_splat(%arg0: tensor<3x3xi32>) -+ - // ----- - -- //////// -++//////// - +// ClampOp - + - +// CHECK-LABEL: func.func @clamp_fold -@@ -103,18 +170,13 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml - + - +// ----- - + --+//////// -+ //////// - // CompareOp - -- // CHECK-LABEL: func.func @compare_folds --@@ -98,6 +113,26 @@ -- // CHECK-DAG: [[R3:%.+]] = stablehlo.constant dense<{{\[\[0, 1, 2, 11, 12\], \[3, 4, 5, 13, 14\]\]}}> : tensor<2x5xi32> -- // CHECK-NEXT: return [[R0]], [[R1]], [[R2]], [[R3]] -- return %0, %1, %2, %3 : tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, tensor<2x5xi32> --+} --+ --+// ----- --+ -+@@ -102,6 +117,26 @@ func.func @concatenate_fold() -> (tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, -+ -+ // ----- -+ - +//////// - +// DivOp - + -@@ -131,32 +193,37 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml - + %1 = stablehlo.divide %cst_1, %cst_1 : tensor - + %2 = stablehlo.divide %cst_2, %cst_2 : tensor - + return %0, %1, %2 : tensor, tensor, tensor -- } -++} -++ -++// ----- -++ -+ //////// -+ // MulOp - -- // ----- --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -+diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -+index 4921a224..a1eed944 100644 -+--- a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir -++++ b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir - @@ -1,4 +1,4 @@ - -// RUN: stablehlo-opt --stablehlo-aggressive-simplification --allow-unregistered-dialect --split-input-file %s | FileCheck %s - +// RUN: stablehlo-opt --stablehlo-aggressive-simplification=fold-op-element-limit=100 --allow-unregistered-dialect --split-input-file %s | FileCheck %s - - ///////// - // AddOp --diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir ----- stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir --+++ stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir --@@ -521,16 +521,16 @@ -+diff --git a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -+index b262bf09..01b5dd8f 100644 -+--- a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -++++ b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir -+@@ -521,16 +521,16 @@ func.func @eval_slice_zerodim() -> tensor<0x2x1xi64> { - // ----- - - // CHECK-LABEL: func @eval_slice_zerorank - -func.func @eval_slice_zerorank() -> tensor { - - // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<3.300000e+01> : tensor --- // CHECK: return [[RESULT]] --- %0 = stablehlo.constant dense<33.0> : tensor - +func.func @eval_slice_zerorank() -> tensor { - + // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<33> : tensor --+ // CHECK: return [[RESULT]] -+ // CHECK: return [[RESULT]] -+- %0 = stablehlo.constant dense<33.0> : tensor - + %0 = stablehlo.constant dense<33> : tensor - %1 = "stablehlo.slice"(%0) { - start_indices = array, -@@ -169,9 +236,10 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b - } - - // ----- --diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir ----- stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir --+++ stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -+diff --git a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -+index 6340898e..a5d344ac 100644 -+--- a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir -++++ b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir - @@ -5,15 +5,9 @@ - // about what constitutes a good test! The CHECK should be - // minimized and named to reflect the test intent. -@@ -190,14 +258,10 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli - // RUN: diff %t.0 %t.1 - - // CHECK-LABEL: vhlo.func_v1 @op_other( --@@ -26,6 +20,34 @@ -- func.func @op_other(%arg0: tensor) -> tensor { -- %0 = arith.addf %arg0, %arg0 : tensor -- return %0 : tensor --+} --+ --+// ----- --+ -+@@ -30,6 +24,34 @@ func.func @op_other(%arg0: tensor) -> tensor { -+ -+ // ----- -+ - +// CHECK-LABEL: vhlo.func_v1 @func_attributes( - +// CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { - +// CHECK: "vhlo.return_v1"(%[[VAL_0]]) : (!vhlo.tensor_v1) -> () -@@ -222,12 +286,17 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli - + vhlo.mixed_dict = {affine_map = affine_map<(d0) -> (d0)>, str_attr = "STR_ATTR"} - +} { - + return %arg0 : tensor -- } -- -- // ----- --diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir ----- stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir --+++ stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -++} -++ -++// ----- -++ -+ // CHECK-LABEL: vhlo.func_v1 @op_shlo( -+ // CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { -+ // CHECK: %[[VAL_1:.*]] = "vhlo.add_v1"(%[[VAL_0]], %[[VAL_0]]) : (!vhlo.tensor_v1, !vhlo.tensor_v1) -> !vhlo.tensor_v1 -+diff --git a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -+index b73d1005..f4330e67 100644 -+--- a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir -++++ b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir - @@ -1,17 +1,8 @@ - // RUN: stablehlo-opt --vhlo-to-version=target=1.9.0 -verify-diagnostics --split-input-file %s - -@@ -248,10 +317,11 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stabl - } { - return - } --diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp ----- stablehlo/stablehlo/tools/StablehloTranslateMain.cpp --+++ stablehlo/stablehlo/tools/StablehloTranslateMain.cpp --@@ -76,6 +76,11 @@ -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index e3171b2d..fdf0d6a9 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -76,6 +76,11 @@ llvm::cl::opt targetOption( - "target", llvm::cl::desc("Target version for serialization"), - llvm::cl::init("")); - -@@ -263,7 +333,7 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st - llvm::cl::opt argsOption( - "args", llvm::cl::desc("Arguments to pass to the interpreter"), - llvm::cl::init("")); --@@ -317,7 +322,8 @@ -+@@ -317,7 +322,8 @@ TranslateFromMLIRRegistration serializeRegistration( - return module.emitError("failed to strip debuginfo"); - } - -@@ -273,10 +343,11 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st - }, - [](DialectRegistry ®istry) { - mlir::registerAllDialects(registry); --diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h ----- stablehlo/stablehlo/transforms/Passes.h --+++ stablehlo/stablehlo/transforms/Passes.h --@@ -35,6 +35,7 @@ -+diff --git a/stablehlo/transforms/Passes.h b/stablehlo/transforms/Passes.h -+index 0e4406d2..a5a4aa27 100644 -+--- a/stablehlo/transforms/Passes.h -++++ b/stablehlo/transforms/Passes.h -+@@ -35,6 +35,7 @@ limitations under the License. - #include "mlir/Transforms/DialectConversion.h" - #include "stablehlo/dialect/StablehloOps.h" - #include "stablehlo/dialect/Version.h" -@@ -284,7 +355,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans - - namespace mlir { - namespace stablehlo { --@@ -54,17 +55,17 @@ -+@@ -54,17 +55,17 @@ void populateStablehloRefineShapesPatterns(MLIRContext *context, - // Populates StableHLO ops to VHLO ops rewriting patterns. - void populateStablehloToVhloPatterns(MLIRContext *context, - RewritePatternSet *patterns, -@@ -305,10 +376,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans - - /// Collection of rewrite patterns for lowering of CHLO ops to StableHLO and - /// Shape ops. --diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp ----- stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp --+++ stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp --@@ -57,7 +57,7 @@ -+diff --git a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -+index e768ded6..25903399 100644 -+--- a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -++++ b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp -+@@ -57,7 +57,7 @@ namespace { - class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - public: - StablehloToVhloTypeConverter(bool allowOtherDialects) -@@ -317,7 +389,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG( - llvm::dbgs() - << "[StablehloToVhloTypeConverter] Creating with allowOtherDialects: " --@@ -82,7 +82,15 @@ -+@@ -82,7 +82,15 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - return vhlo::TokenV1Type::get(token.getContext()); - }); - addBuiltinToVhloConversions(); -@@ -334,7 +406,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertEncoding(Attribute attr) const final { --@@ -114,7 +122,7 @@ -+@@ -114,7 +122,7 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { - return vhlo::Name##Version##Attr::get(attr.getContext(), vhloValue.value()) - - Attribute convertGeneric(Attribute stablehloAttr, -@@ -343,7 +415,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Convert generic: " << stablehloAttr << '\n'); - - // Handle StableHLO attributes. --@@ -241,6 +249,10 @@ -+@@ -241,6 +249,10 @@ Attribute convertGeneric(Attribute stablehloAttr, - return vhlo::TypeV1Attr::get(attr.getContext(), vhloType); - } - -@@ -354,7 +426,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Failed to convert: " << stablehloAttr << '\n'); - return {}; // Failed to convert attribute. - } --@@ -268,13 +280,15 @@ -+@@ -268,13 +280,15 @@ SpecialResult notSpecial() { return SpecialResult::NOT_SPECIAL; } - Attribute convertBool(const ConversionPattern& pattern, int64_t stablehloDim) { - auto stablehloType = IntegerType::get(pattern.getContext(), 1); - auto stablehloAttr = IntegerAttr::get(stablehloType, stablehloDim); -@@ -372,7 +444,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertInts(const ConversionPattern& pattern, --@@ -282,7 +296,8 @@ -+@@ -282,7 +296,8 @@ Attribute convertInts(const ConversionPattern& pattern, - auto stablehloType = RankedTensorType::get( - stablehloDims.size(), IntegerType::get(pattern.getContext(), 64)); - auto stablehloAttr = DenseIntElementsAttr::get(stablehloType, stablehloDims); -@@ -382,7 +454,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - Attribute convertSymbol(const ConversionPattern& pattern, --@@ -290,7 +305,7 @@ -+@@ -290,7 +305,7 @@ Attribute convertSymbol(const ConversionPattern& pattern, - auto stablehloSymbolAttr = dyn_cast(stablehloAttr); - if (!stablehloSymbolAttr) return {}; - return convertGeneric(stablehloSymbolAttr.getAttr(), -@@ -391,7 +463,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - } - - SpecialResult convertChannelHandle(const ConversionPattern& pattern, --@@ -445,15 +460,15 @@ -+@@ -445,15 +460,15 @@ SpecialResult convertDotAlgorithm(const ConversionPattern& pattern, - vhloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "lhs_precision_type"), - convertGeneric(TypeAttr::get(attr.getLhsPrecisionType()), -@@ -410,7 +482,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - - // Components - auto vhloLhsComponentCount = convertInt(pattern, attr.getLhsComponentCount()); --@@ -712,7 +727,9 @@ -+@@ -712,7 +727,9 @@ LogicalResult addDefaults(const OpConversionPattern& pattern, - auto addDefaultAttr = [&](StringRef vhloName, Attribute stablehloAttr) { - vhloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), vhloName), -@@ -421,7 +493,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - }; - if constexpr (std::is_same::value) { - if (!stablehloOp.getSymVisibilityAttr()) --@@ -987,8 +1004,9 @@ -+@@ -987,8 +1004,9 @@ class StablehloToVhloOpConverter : public OpConversionPattern { - case SpecialResult::SPECIAL_FAILURE: - return failure(); - case SpecialResult::NOT_SPECIAL: -@@ -433,7 +505,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - if (!vhloAttr) return failure(); - vhloAttrs.push_back({stablehloAttr.getName(), vhloAttr}); - break; --@@ -1075,14 +1093,14 @@ -+@@ -1075,14 +1093,14 @@ struct StablehloLegalizeToVhloPass - } - - private: -@@ -450,10 +522,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable - populateStablehloToVhloPatterns< - #define GET_OP_LIST - #include "stablehlo/dialect/StablehloOps.cpp.inc" --diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp ----- stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp --+++ stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp --@@ -16,6 +16,7 @@ -+diff --git a/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/transforms/StablehloRefineShapes.cpp -+index 64eaa3c0..4b585b46 100644 -+--- a/stablehlo/transforms/StablehloRefineShapes.cpp -++++ b/stablehlo/transforms/StablehloRefineShapes.cpp -+@@ -16,6 +16,7 @@ limitations under the License. - - #include - #include -@@ -461,7 +534,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl - #include - #include - --@@ -1038,8 +1039,11 @@ -+@@ -1038,8 +1039,11 @@ LogicalResult applyShapeRefinementPatterns(func::FuncOp func, - // Populate additional patterns for StableHLO extensions. - state.addAdditionalPatterns(patterns); - -@@ -473,10 +546,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl - - // The folding patterns implement partial evaluation of shape computations - // which is a critical part of implementing type refinement for ops like --diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp ----- stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp --+++ stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp --@@ -57,7 +57,10 @@ -+diff --git a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -+index 5662ad9d..41973a0e 100644 -+--- a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -++++ b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp -+@@ -57,7 +57,10 @@ namespace { - - class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { - public: -@@ -488,7 +562,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - addConversion([](Type type) -> Type { return type; }); - addConversion([](vhlo::TokenV1Type token) -> Type { - LLVM_DEBUG(llvm::dbgs() << "Converting TokenType\n"); --@@ -90,7 +93,7 @@ -+@@ -90,7 +93,7 @@ class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { - return stablehlo::Name##Attr::get(attr.getContext(), stablehloValue.value()) - - Attribute convertGeneric(Attribute vhloAttr, -@@ -497,18 +571,18 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - LLVM_DEBUG(llvm::dbgs() << "Converting attr " << vhloAttr); - if (auto vhloAttrs = dyn_cast(vhloAttr)) { - SmallVector stablehloAttrs; --@@ -189,6 +192,10 @@ -- // All VHLO attributes must have counterparts in StableHLO. -+@@ -190,6 +193,10 @@ Attribute convertGeneric(Attribute vhloAttr, - return {}; - } --+ -+ - + // Fall back to type converter for unknown attributes. - + auto unknownAttr = typeConverter->convertUnknownAttribute(vhloAttr); - + if (unknownAttr) return unknownAttr; -- -++ - // This should be unreachable unless program is a mix of VHLO and other - // dialects, e.g. due to user edits to textual assembly format. --@@ -229,7 +236,7 @@ -+ return {}; -+@@ -229,7 +236,7 @@ bool isNoneType(Attribute vhloAttr) { - } - - LogicalResult convertTypeAttr(Attribute vhloAttr, Type& result, -@@ -517,7 +591,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - auto stablehloAttr = convertGeneric(vhloAttr, typeConverter); - if (!stablehloAttr || !isa(stablehloAttr)) return failure(); - result = cast(stablehloAttr).getValue(); --@@ -244,7 +251,7 @@ -+@@ -244,7 +251,7 @@ LogicalResult convertInt(Attribute vhloAttr, int64_t& result) { - } - - LogicalResult convertInts(Attribute vhloAttr, -@@ -526,7 +600,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector& result) { - auto vhloTensorAttr = dyn_cast(vhloAttr); - if (!vhloTensorAttr) return failure(); --@@ -256,7 +263,7 @@ -+@@ -256,7 +263,7 @@ LogicalResult convertInts(Attribute vhloAttr, - } - - Attribute convertSymbol(Attribute vhloAttr, -@@ -535,7 +609,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - auto vhloStringAttr = dyn_cast(vhloAttr); - if (!vhloStringAttr) return {}; - auto stablehloStringAttr = dyn_cast_or_null( --@@ -267,7 +274,7 @@ -+@@ -267,7 +274,7 @@ Attribute convertSymbol(Attribute vhloAttr, - - template - Attribute convertChannelHandle(OpType vhloOp, -@@ -544,7 +618,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t channelId, channelType; - if (failed(convertInt(vhloOp.getChannelId(), channelId)) || - failed(convertInt(vhloOp.getChannelType(), channelType))) --@@ -277,7 +284,7 @@ -+@@ -277,7 +284,7 @@ Attribute convertChannelHandle(OpType vhloOp, - } - - Attribute convertChannelId(Attribute vhloAttr, -@@ -553,7 +627,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t channelId; - if (failed(convertInt(vhloAttr, channelId))) return {}; - return stablehlo::ChannelHandleAttr::get(vhloAttr.getContext(), channelId, --@@ -285,8 +292,8 @@ -+@@ -285,8 +292,8 @@ Attribute convertChannelId(Attribute vhloAttr, - } - - template -@@ -564,7 +638,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - int64_t stablehloInputBatchDimension, stablehloInputFeatureDimension; - SmallVector stablehloInputSpatialDimensions; - int64_t stablehloKernelInputFeatureDimension, --@@ -323,7 +330,7 @@ -+@@ -323,7 +330,7 @@ Attribute convertConvDimensionNumbers(OpType vhloOp, - } - - Attribute convertCustomCallCalledComputations( -@@ -573,7 +647,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (auto vhloArrayAttr = dyn_cast(vhloAttr)) { - SmallVector stablehloAttrs; - for (auto vhloAttr : vhloArrayAttr.getValue()) { --@@ -336,8 +343,8 @@ -+@@ -336,8 +343,8 @@ Attribute convertCustomCallCalledComputations( - return {}; - } - -@@ -584,7 +658,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - Type lhsPrecisionType, rhsPrecisionType, accumulationType; - if (isNoneType(vhloOp.getLhsComponentCount())) { - // All must be nonetype --@@ -373,8 +380,8 @@ -+@@ -373,8 +380,8 @@ FailureOr convertDotAlgorithm(vhlo::DotGeneralOpV2 vhloOp, - numPrimitiveOperations, allowImpreciseAccumulation); - } - -@@ -595,7 +669,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloLhsBatchingDimensions, - stablehloRhsBatchingDimensions, stablehloLhsContractingDimensions, - stablehloRhsContractingDimensions; --@@ -394,13 +401,13 @@ -+@@ -394,13 +401,13 @@ Attribute convertDotDimensionNumbers(vhlo::DotGeneralOpV2 vhloOp, - } - - Attribute convertFuncCallee(Attribute vhloAttr, -@@ -612,7 +686,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloOffsetDims, stablehloCollapsedSliceDims, - stablehloOperandBatchingDims, stablehloStartIndicesBatchingDims, - stablehloStartIndexMap; --@@ -423,8 +430,8 @@ -+@@ -423,8 +430,8 @@ Attribute convertGatherDimensionNumbers(OpType vhloOp, - stablehloStartIndexMap, stablehloIndexVectorDim); - } - -@@ -623,7 +697,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - SmallVector stablehloUpdateWindowDims, stablehloInsertedWindowDims, - stablehloInputBatchingDims, stablehloScatterIndicesBatchingDims, - stablehloScatterDimsToOperandDims; --@@ -463,10 +470,11 @@ -+@@ -463,10 +470,11 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - VhloOpTy vhloOp, - SmallVector& vhloAttrs, - SmallVector& stablehloAttrs) { -@@ -637,7 +711,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dimension_numbers"), --@@ -480,7 +488,7 @@ -+@@ -480,7 +488,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - if constexpr (std::is_same::value) { - // Dot Dimension Numbers - auto stablehloDotDimAttr = -@@ -646,7 +720,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloDotDimAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dot_dimension_numbers"), --@@ -489,8 +497,7 @@ -+@@ -489,8 +497,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - "lhs_contracting_dimensions", "rhs_contracting_dimensions"); - - // Dot Algorithm -@@ -656,7 +730,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (failed(stablehloDotAlgorithmAttr)) return failure(); - if (stablehloDotAlgorithmAttr.value()) - stablehloAttrs.emplace_back( --@@ -503,8 +510,7 @@ -+@@ -503,8 +510,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value || - std::is_same::value) { -@@ -666,7 +740,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "dimension_numbers"), --@@ -514,8 +520,7 @@ -+@@ -514,8 +520,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - "start_index_map", "index_vector_dim"); - } - if constexpr (std::is_same::value) { -@@ -676,7 +750,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "scatter_dimension_numbers"), --@@ -526,8 +531,7 @@ -+@@ -526,8 +531,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value || - std::is_same::value) { -@@ -686,7 +760,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.emplace_back( - StringAttr::get(pattern.getContext(), "channel_handle"), stablehloAttr); --@@ -537,7 +541,7 @@ -+@@ -537,7 +541,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, - } - - template -@@ -695,7 +769,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - StringAttr vhloName, Attribute vhloAttr, - SmallVector& stablehloAttrs) { - auto tensorAttr = dyn_cast(vhloAttr); --@@ -556,15 +560,15 @@ -+@@ -556,15 +560,15 @@ SpecialResult convertDenseArray(const TypeConverter* typeConverter, - } - - SpecialResult convertDenseBoolArray( -@@ -715,7 +789,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - return convertDenseArray( - typeConverter, vhloName, vhloAttr, stablehloAttrs); - } --@@ -575,7 +579,8 @@ -+@@ -575,7 +579,8 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - SmallVector& stablehloAttrs) { - StringAttr stablehloName = vhloName; - Attribute stablehloAttr; -@@ -725,7 +799,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - - if constexpr (std::is_same::value || - std::is_same::value || --@@ -585,7 +590,7 @@ -+@@ -585,7 +590,7 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - std::is_same::value) { - if (vhloName == "channel_id") { - stablehloName = StringAttr::get(pattern.getContext(), "channel_handle"); -@@ -734,7 +808,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return specialFailure(); - } - if (vhloName == "use_global_device_ids") { --@@ -597,20 +602,20 @@ -+@@ -597,20 +602,20 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, - } - if constexpr (std::is_same::value) { - if (vhloName == "called_computations") { -@@ -759,7 +833,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return specialFailure(); - } - } --@@ -760,8 +765,8 @@ -+@@ -760,8 +765,8 @@ bool isDefaultResultAccuracyAttribute(Attribute vhloAttr) { - template - bool isSplatTensor(const ConversionPattern& pattern, Attribute vhloAttr, - T splatValue) { -@@ -770,7 +844,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - return attr && attr.isSplat() && - attr.template getSplatValue() == splatValue; - } --@@ -977,8 +982,9 @@ -+@@ -977,8 +982,9 @@ class VhloToStablehloOpConverter : public OpConversionPattern { - case SpecialResult::SPECIAL_FAILURE: - return failure(); - case SpecialResult::NOT_SPECIAL: -@@ -782,7 +856,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - if (!stablehloAttr) return failure(); - stablehloAttrs.push_back({vhloAttr.getName(), stablehloAttr}); - break; --@@ -1056,7 +1062,7 @@ -+@@ -1056,7 +1062,7 @@ struct ReconcileUnrealizedConversionCasts - template - void populateVhloToStablehloPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -791,7 +865,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - patterns - ->add>...>( - *converter, context); --@@ -1104,7 +1110,7 @@ -+@@ -1104,7 +1110,7 @@ struct VhloLegalizeToStablehloPass - - void populateVhloToStablehloPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -800,10 +874,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable - populateVhloToStablehloPatterns< - #define GET_OP_LIST - #include "stablehlo/dialect/StablehloOps.cpp.inc" --diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stablehlo/transforms/VhloToVersion.cpp ----- stablehlo/stablehlo/transforms/VhloToVersion.cpp --+++ stablehlo/stablehlo/transforms/VhloToVersion.cpp --@@ -56,15 +56,23 @@ -+diff --git a/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/transforms/VhloToVersion.cpp -+index dfbf2877..7818e601 100644 -+--- a/stablehlo/transforms/VhloToVersion.cpp -++++ b/stablehlo/transforms/VhloToVersion.cpp -+@@ -56,15 +56,23 @@ namespace { - - // Currently there are no type-to-version conversions so this class - // simply validates that all types are from the VHLO dialect. -@@ -833,7 +908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - }; - - // Check user-specified target version. Emit error if invalid. --@@ -111,6 +119,10 @@ -+@@ -111,6 +119,10 @@ bool isLegalVersion(VersionedInterface& interface, const Version& target) { - LogicalResult isLegalType(Type type, const Version& targetVersion); - - LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { -@@ -844,7 +919,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - auto attrInterface = dyn_cast(attr); - if (!attrInterface || !isLegalVersion(attrInterface, targetVersion)) { - LLVM_DEBUG(llvm::dbgs() << "failed to legalize attribute " << attr --@@ -119,10 +131,11 @@ -+@@ -119,10 +131,11 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { - } - - // Recursively check attrs if VHLO attr is a container -@@ -857,7 +932,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (auto arrAttr = dyn_cast(attr)) { - return success(llvm::all_of( - arrAttr.getValue(), [&](std::pair entry) { --@@ -146,6 +159,10 @@ -+@@ -146,6 +159,10 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { - } - - LogicalResult isLegalType(Type type, const Version& targetVersion) { -@@ -868,7 +943,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - // All valid VHLO types must have versioned type interface. - auto typeInterface = dyn_cast(type); - if (!typeInterface || !isLegalVersion(typeInterface, targetVersion)) { --@@ -170,10 +187,11 @@ -+@@ -170,10 +187,11 @@ LogicalResult isLegalType(Type type, const Version& targetVersion) { - return failure(); - return isLegalType(ranked.getElementType(), targetVersion); - } -@@ -881,7 +956,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (auto quant = dyn_cast(type)) - return success( - succeeded(isLegalType(quant.getStorageType(), targetVersion)) && --@@ -213,7 +231,7 @@ -+@@ -213,7 +231,7 @@ bool isLegalLocation(Location loc, const Version& targetVersion) { - bool isLegalOperation(Operation* op, const Version& targetVersion) { - // Validate op - auto opInterface = dyn_cast(op); -@@ -890,7 +965,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - if (!isLegalVersion(opInterface, targetVersion)) return false; - LLVM_DEBUG(llvm::dbgs() << "Legal op version for target. " << op << '\n'); - --@@ -454,7 +472,7 @@ -+@@ -454,7 +472,7 @@ struct AllReduceOpV2ToV1 : public OpRewritePattern { - namespace stablehlo { - void populateVhloToVersionPatterns(MLIRContext* context, - RewritePatternSet* patterns, -@@ -899,10 +974,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable - vhlo::populateWithGenerated(*patterns); - patterns->add(context); - patterns->add(context); --diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp ----- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp --+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp --@@ -19,6 +19,7 @@ -+diff --git a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -+index 9cfac0ba..09bfb783 100644 -+--- a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -++++ b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp -+@@ -19,6 +19,7 @@ limitations under the License. - #include - #include - #include -@@ -910,7 +986,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - #include - - #include "llvm/ADT/APInt.h" --@@ -47,6 +48,7 @@ -+@@ -47,6 +48,7 @@ limitations under the License. - #include "mlir/IR/TypeUtilities.h" - #include "mlir/IR/Types.h" - #include "mlir/IR/Value.h" -@@ -918,7 +994,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - #include "mlir/Interfaces/SideEffectInterfaces.h" - #include "mlir/Pass/Pass.h" - #include "mlir/Rewrite/FrozenRewritePatternSet.h" --@@ -98,29 +100,88 @@ -+@@ -98,29 +100,88 @@ LogicalResult validateStaticShapeResult(PatternRewriter& rewriter, - return success(); - } - -@@ -982,8 +1058,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - if (res) return cast(res); - - return nullptr; --+} --+ -+ } -+ - + - +/// Binary constant folder that used a generic folder function to handle both - +/// ints and floats. -@@ -1005,8 +1081,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (!res) return rewriter.notifyMatchFailure(op, "folding failed"); - + - + return res; -- } -- -++} -++ - template - -LogicalResult evalConvertHelper(PatternRewriter& rewriter, OpType op, -@@ -1014,7 +1090,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr elements, Type resType, - CalculationT&& calculate) { - auto result = constFoldCastOp( - elements, resType, calculate); - -@@ -1036,7 +1112,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr elements, - RankedTensorType resultType) { - auto oldType = getElementTypeOrSelf(elements); --@@ -153,7 +215,7 @@ -+@@ -153,7 +215,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - if (auto newFloatType = dyn_cast(newType)) { - // Float -> Float - const auto& targetSemantics = newFloatType.getFloatSemantics(); -@@ -1045,7 +1121,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&targetSemantics](const APFloat& operand, bool& castStatus) { - bool losesInfo; --@@ -167,7 +229,7 @@ -+@@ -167,7 +229,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - } - - // Float -> Int -@@ -1054,7 +1130,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newBitWidth, &isNewTypeUnsigned](const APFloat& operand, - bool& castStatus) { --@@ -186,7 +248,7 @@ -+@@ -186,7 +248,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - - if (auto newFloatType = dyn_cast(newType)) { - // Int -> Float -@@ -1063,7 +1139,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newFloatType, &isOldTypeUnsigned](const APInt& operand, - bool& /*castStatus*/) { --@@ -199,64 +261,12 @@ -+@@ -199,7 +261,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - } - - // Int -> Int -@@ -1072,10 +1148,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - rewriter, op, elements, resultType, - [&newBitWidth, &isOldTypeUnsigned](const APInt& operand, - bool& /*castStatus*/) { -- return APSInt(operand, isOldTypeUnsigned).extOrTrunc(newBitWidth); -+@@ -207,58 +269,6 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, - }); ---} --- -+ } -+ - -// The patterns below implement partial evaluation of shape computations which - -// is a critical part of implementing type refinement for ops like - -// dynamic_broadcast_in_dim, dynamic_iota and dynamic_reshape whose shape -@@ -1126,10 +1202,12 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - rewriter.replaceOpWithNewOp(op, - - getTensorAttr(resultType, result)); - - return success(); -- } -- -+-} -+- - template --@@ -275,29 +285,18 @@ -+ struct FoldOpRewritePattern : OpRewritePattern { -+ FoldOpRewritePattern(MLIRContext* context, -+@@ -275,29 +285,18 @@ struct FoldOpRewritePattern : OpRewritePattern { - ArrayRef generatedNames = {}) = delete; - - const StablehloAggressiveFolderPassOptions& options; -@@ -1154,9 +1232,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - rewriter.replaceOpWithNewOp(op, res); - - return success(); - - } --- -+ - - return failure(); --+ - + LogicalResult validateElementCountForFold(PatternRewriter& rewriter, - + Operation* op, - + ShapedType resultType) const { -@@ -1171,31 +1248,21 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - }; - --@@ -318,100 +317,102 @@ -+@@ -318,100 +317,102 @@ struct ShapeOpRewritePattern : public FoldOpRewritePattern { - } - }; - - -struct EvalAddOpShapePattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- --- LogicalResult matchAndRewrite(AddOp op, --- PatternRewriter& rewriter) const override { --- return evalElementwise(rewriter, op, --- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); --- } ---}; --- ---struct EvalAndOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; --- --- LogicalResult matchAndRewrite(AndOp op, --- PatternRewriter& rewriter) const override { - +struct FoldAddOpPattern final - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(AddOp op, - + LogicalResult matchAndRewrite(mlir::stablehlo::AddOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { -+- return evalElementwise(rewriter, op, -+- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); - + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) - + return failure(); - + -@@ -1203,14 +1270,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalAndOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldAndOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(AndOp op, - + LogicalResult matchAndRewrite(mlir::stablehlo::AndOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { - + // TODO: Support more int types - auto resultType = op.getType(); - if (!resultType.getElementType().isInteger(1)) -@@ -1219,24 +1289,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { - - return getAPSInt(resultType.getElementType(), lhsInt != 0 && rhsInt != 0); - - }); --- } - + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldAnd{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+ --+ struct FoldAnd { --+ APInt operator()(APInt lhs, APInt rhs) const { --+ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); --+ } --+ std::optional operator()(APFloat lhs, APFloat rhs) const { --+ return std::nullopt; --+ } --+ }; -- }; -+ } -+-}; - -- // Pattern: broadcast_in_dim(splat, _) -> constant(splat) -+-// Pattern: broadcast_in_dim(splat, _) -> constant(splat) - -struct FoldBroadcastInDimSplatPattern final - - : FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; -@@ -1251,14 +1311,22 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - op, SplatElementsAttr::get(op.getType(), - - cstAttr.getSplatValue())); - - return success(); --- } -++ struct FoldAnd { -++ APInt operator()(APInt lhs, APInt rhs) const { -++ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); -+ } - - return failure(); - - } ---}; --- -++ std::optional operator()(APFloat lhs, APFloat rhs) const { -++ return std::nullopt; -++ } -++ }; -+ }; -+ - -struct EvalBroadcastInDimOpPattern - - : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; -++// Pattern: broadcast_in_dim(splat, _) -> constant(splat) - +struct FoldBroadcastInDimOpSplatPattern - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; -@@ -1267,8 +1335,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - auto resultType = op.getType(); - - if (failed(validateStaticShapeResult(rewriter, op, resultType))) --- return failure(); --- -++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || -++ failed(validateShapeFoldDtype(rewriter, op, resultType))) -+ return failure(); -+ - - auto operandType = op.getOperand().getType(); - - if (operandType.getRank() != 0) - - return rewriter.notifyMatchFailure(op, "expected 0-dimensional type"); -@@ -1277,52 +1347,34 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - if (failed(hlo::matchInts(op.getOperand(), operand))) - - return rewriter.notifyMatchFailure(op, "expected constant operands"); - - auto scalar = operand[0]; --- -++ SplatElementsAttr cstAttr; -++ matchPattern(op.getOperand(), m_Constant(&cstAttr)); -++ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); -+ - - rewriter.replaceOpWithNewOp( - - op, getTensorAttr(op.getType(), scalar)); --- return success(); --- } ---}; --- -++ rewriter.replaceOpWithNewOp( -++ op, SplatElementsAttr::get(op.getType(), -++ cstAttr.getSplatValue())); -+ return success(); -+ } -+ }; -+ - -struct EvalClampOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++struct FoldCompareOpPattern : public ShapeOpRewritePattern { -++ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ - - LogicalResult matchAndRewrite(ClampOp op, --- PatternRewriter& rewriter) const override { -++ LogicalResult matchAndRewrite(CompareOp op, -+ PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt min, APSInt operand, APSInt max) { - - if (operand < min) return min; - - if (max < operand) return max; - - return operand; - - }); --- } ---}; --- ---struct EvalCompareOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; --+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || --+ failed(validateShapeFoldDtype(rewriter, op, resultType))) --+ return failure(); --+ --+ SplatElementsAttr cstAttr; --+ matchPattern(op.getOperand(), m_Constant(&cstAttr)); --+ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); --+ --+ rewriter.replaceOpWithNewOp( --+ op, SplatElementsAttr::get(op.getType(), --+ cstAttr.getSplatValue())); --+ return success(); --+ } --+}; --+ --+struct FoldCompareOpPattern : public ShapeOpRewritePattern { --+ using ShapeOpRewritePattern::ShapeOpRewritePattern; -- -- LogicalResult matchAndRewrite(CompareOp op, -- PatternRewriter& rewriter) const override { -- auto resultType = op.getType(); --- auto kind = op.getCompareType(); --- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { -++ auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); - + -@@ -1332,15 +1384,23 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+ -+ } -+-}; -+ -+-struct EvalCompareOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - + struct FoldCompare { - + FoldCompare(ComparisonDirection direction, - + std::optional kind) - + : direction(direction), kind(kind) {} - + ComparisonDirection direction; - + std::optional kind; --+ -+ -+- LogicalResult matchAndRewrite(CompareOp op, -+- PatternRewriter& rewriter) const override { -+- auto resultType = op.getType(); -+- auto kind = op.getCompareType(); -+- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { - + // TODO: Enable float folding. - + std::optional operator()(APFloat lhs, APFloat rhs) { - + return std::nullopt; -@@ -1352,7 +1412,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - case ComparisonDirection::EQ: - result = lhs == rhs; - break; --@@ -431,9 +432,9 @@ -+@@ -431,9 +432,9 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { - result = kind == ComparisonType::SIGNED ? lhs.slt(rhs) : lhs.ult(rhs); - break; - } -@@ -1365,7 +1425,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - }; - - ////////////////////////////////// --@@ -441,16 +442,15 @@ -+@@ -441,16 +442,15 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { - ///////////////////////////////// - - struct FoldConcatenateOpPattern final -@@ -1387,7 +1447,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - // Fold concatenate when all inputs are constants. --@@ -466,6 +466,7 @@ -+@@ -466,6 +466,7 @@ struct FoldConcatenateOpPattern final - int64_t{1}, std::multiplies<>{}); - - SmallVector newElems; -@@ -1395,7 +1455,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - newElems.reserve(numElems); - - for (int64_t i = 0; i != topSize; ++i) { --@@ -485,31 +486,7 @@ -+@@ -485,31 +486,7 @@ struct FoldConcatenateOpPattern final - int64_t foldOpElementLimit; - }; - -@@ -1428,20 +1488,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - using ShapeOpRewritePattern::ShapeOpRewritePattern; - - LogicalResult matchAndRewrite(ConvertOp op, --@@ -532,28 +509,50 @@ -+@@ -532,28 +509,50 @@ struct EvalConvertOpPattern : public ShapeOpRewritePattern { - return rewriter.notifyMatchFailure( - op, "expected constant integer or float operand"); - - - return evalConvert(rewriter, op, elements, resultType); --- } ---}; --- -++ return foldConvert(rewriter, op, elements, resultType); -+ } -+ }; -+ - -struct EvalDivOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --+ return foldConvert(rewriter, op, elements, resultType); --+ } --+}; --+ - +struct FoldDivOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1449,12 +1506,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs / rhs; }); --- } ---}; --- ---struct EvalGetDimensionSizeOpPattern --- : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1464,7 +1515,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldDivide { - + FoldDivide(bool isUnsignedInt) -@@ -1479,8 +1530,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.udiv(rhs); } - + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.sdiv(rhs); } - + }; --+}; --+ -+ }; -+ -+-struct EvalGetDimensionSizeOpPattern -+- : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldGetDimensionSizeOpPattern - + : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; -@@ -1494,7 +1548,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - auto operandType = op.getOperand().getType(); --@@ -567,86 +566,187 @@ -+@@ -567,86 +566,187 @@ struct EvalGetDimensionSizeOpPattern - } - }; - -@@ -1549,11 +1603,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { - - return lhs >= rhs ? lhs : rhs; - - }); --- } ---}; --- ---struct EvalMinOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1563,9 +1612,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalMinOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldMinOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1574,11 +1625,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { - - return lhs <= rhs ? lhs : rhs; - - }); --- } ---}; --- ---struct FoldMulOpPattern final : FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1588,19 +1634,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct FoldMulOpPattern final : FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +// Clamp is folded using Min and Max folders. - +struct FoldClampOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; --+ -+ -+- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, - + LogicalResult matchAndRewrite(ClampOp op, --+ PatternRewriter& rewriter) const override { -+ PatternRewriter& rewriter) const override { -+- TypedAttr lhsAttr; -+- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); -+- -+- TypedAttr rhsAttr; -+- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); -+- -+- if (TypedAttr res; -+- lhsAttr && rhsAttr && -+- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { -+- rewriter.replaceOpWithNewOp(op, res); -+- return success(); -+- } - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); --+ -+ -+- return failure(); - + TypedAttr minAttr, operandAttr, maxAttr; - + matchPattern(op.getMin(), m_Constant(&minAttr)); - + matchPattern(op.getOperand(), m_Constant(&operandAttr)); -@@ -1620,43 +1682,19 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (!res) return rewriter.notifyMatchFailure(op, "failed to fold clamp"); - + rewriter.replaceOpWithNewOp(op, res); - + return success(); --+ } --+}; --+ --+struct FoldMulOpPattern final : ShapeOpRewritePattern { --+ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ } -+ }; - -- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, -- PatternRewriter& rewriter) const override { --- TypedAttr lhsAttr; --- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); --- --- TypedAttr rhsAttr; --- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); --- --- if (TypedAttr res; --- lhsAttr && rhsAttr && --- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { --- rewriter.replaceOpWithNewOp(op, res); --- return success(); --- } --- --- return failure(); --- } ---}; --- - -struct EvalMulOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++struct FoldMulOpPattern final : ShapeOpRewritePattern { -++ using ShapeOpRewritePattern::ShapeOpRewritePattern; -+ - - LogicalResult matchAndRewrite(MulOp op, --- PatternRewriter& rewriter) const override { -++ LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, -+ PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs * rhs; }); --- } ---}; --- ---struct EvalOrOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) - + return failure(); - + -@@ -1664,9 +1702,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } --+}; --+ -+ } -+ }; -+ -+-struct EvalOrOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldOrOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1680,16 +1720,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { - - return getAPSInt(resultType.getElementType(), lhsInt != 0 || rhsInt != 0); - - }); --- } ---}; --- ---struct EvalRemOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldOr{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldOr { - + APInt operator()(APInt lhs, APInt rhs) const { -@@ -1699,8 +1734,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return std::nullopt; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalRemOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldRemOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1708,10 +1745,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs % rhs; }); --- } ---}; --- ---struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - + auto resultType = op.getType(); - + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) - + return failure(); -@@ -1721,7 +1754,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); - + return success(); --+ } -+ } - + - + struct FoldRem { - + FoldRem(bool isUnsignedInt) -@@ -1736,14 +1769,15 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.urem(rhs); } - + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.srem(rhs); } - + }; --+}; --+ -+ }; -+ -+-struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - +// Pattern: reshape(cst, shape) -> cst - +struct FoldReshapeOpPattern : public ShapeOpRewritePattern { - using ShapeOpRewritePattern::ShapeOpRewritePattern; - - LogicalResult matchAndRewrite(ReshapeOp op, --@@ -656,7 +756,6 @@ -+@@ -656,7 +756,6 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - failed(validateShapeFoldDtype(rewriter, op, resultType))) - return failure(); - -@@ -1751,7 +1785,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseIntOrFPElementsAttr attr; - if (!matchPattern(op.getOperand(), m_Constant(&attr))) - return rewriter.notifyMatchFailure(op, "expected constant operand"); --@@ -665,53 +764,98 @@ -+@@ -665,53 +764,98 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { - } - }; - -@@ -1764,16 +1798,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - PatternRewriter& rewriter) const override { - auto resultType = op.getType(); - - if (failed(validateStaticShapeResult(rewriter, op, resultType))) --- return failure(); --- -++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || -++ failed(validateShapeFoldDtype(rewriter, op, resultType))) -+ return failure(); -+ - - SmallVector pred, onTrue, onFalse; - - if (failed(hlo::matchInts(op.getPred(), pred)) || - - failed(hlo::matchInts(op.getOnTrue(), onTrue)) || - - failed(hlo::matchInts(op.getOnFalse(), onFalse))) --+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || --+ failed(validateShapeFoldDtype(rewriter, op, resultType))) --+ return failure(); --+ - + DenseIntElementsAttr predAttr; - + DenseElementsAttr onTrueAttr, onFalseAttr; - + matchPattern(op.getPred(), m_Constant(&predAttr)); -@@ -1783,14 +1815,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return rewriter.notifyMatchFailure(op, "expected constant operands"); - - - SmallVector result; -+- for (auto [predEl, onTrueEl, onFalseEl] : -+- llvm::zip(pred, onTrue, onFalse)) { -+- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); - + // Optimization, handle splat predicate - + if (isa(predAttr)) { - + auto pred = predAttr.getSplatValue(); - + rewriter.replaceOpWithNewOp( - + op, pred.isZero() ? onFalseAttr : onTrueAttr); - + return success(); --+ } --+ -+ } -+ - + // TODO: Enable float folding. - + if (op.getType().getElementType().isFloat()) - + return rewriter.notifyMatchFailure(op, "float select not supported yet"); -@@ -1800,27 +1835,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return failure(); - + - + SmallVector result; -- for (auto [predEl, onTrueEl, onFalseEl] : --- llvm::zip(pred, onTrue, onFalse)) { --- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); --- } --- -++ for (auto [predEl, onTrueEl, onFalseEl] : - + llvm::zip(predAttr.getValues(), onTrueAttr.getValues(), - + onFalseAttr.getValues())) { - + result.push_back(!predEl.isZero() ? onTrueEl : onFalseEl); - + } - rewriter.replaceOpWithNewOp( - - op, getTensorAttr(op.getType(), result)); --- return success(); --- } ---}; --- ---struct EvalSignOpPattern : public FoldOpRewritePattern { --- using FoldOpRewritePattern::FoldOpRewritePattern; - + op, DenseIntElementsAttr::get(resultType, result)); - + --+ return success(); --+ } -+ return success(); -+ } - + - + struct FoldSelect { - + std::optional operator()(APFloat pred, APFloat onTrue, -@@ -1832,8 +1857,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - + return pred != 0 ? onTrue : onFalse; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalSignOpPattern : public FoldOpRewritePattern { -+- using FoldOpRewritePattern::FoldOpRewritePattern; - +struct FoldSignOpPattern : public ShapeOpRewritePattern { - + using ShapeOpRewritePattern::ShapeOpRewritePattern; - -@@ -1881,7 +1908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - }; - - template --@@ -749,13 +893,14 @@ -+@@ -749,13 +893,14 @@ DenseElementsAttr sliceType(SliceOp& op, const RangeType& data) { - ArrayRef(result)); - } - -@@ -1899,7 +1926,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - return failure(); - - auto operand = op.getOperand(); --@@ -784,36 +929,18 @@ -+@@ -784,36 +929,18 @@ struct EvalSliceOpPattern : public FoldOpRewritePattern { - }; - - struct FoldSubtractOpPattern final -@@ -1930,14 +1957,13 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - - -struct EvalSubtractOpPattern : public FoldOpRewritePattern { - - using FoldOpRewritePattern::FoldOpRewritePattern; --- -++ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) -++ return failure(); -+ - - LogicalResult matchAndRewrite(SubtractOp op, - - PatternRewriter& rewriter) const override { - - return evalElementwise(rewriter, op, - - [&](APSInt lhs, APSInt rhs) { return lhs - rhs; }); --+ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) --+ return failure(); --+ - + auto res = foldBinaryOpIntOrFloat(rewriter, op, std::minus<>{}); - + if (failed(res)) return failure(); - + rewriter.replaceOpWithNewOp(op, res.value()); -@@ -1945,23 +1971,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - }; - --@@ -823,42 +950,38 @@ -+@@ -823,42 +950,38 @@ struct FoldSqrtOpPattern - - LogicalResult matchAndRewrite(mlir::stablehlo::SqrtOp op, - PatternRewriter& rewriter) const final { - - TypedAttr lhsAttr; - - matchPattern(op.getOperand(), m_Constant(&lhsAttr)); --- -++ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); -++ if (failed(res)) return failure(); -++ rewriter.replaceOpWithNewOp(op, res.value()); -++ return success(); -++ } -+ - - if (!lhsAttr) - - return rewriter.notifyMatchFailure(op, "operand not constant"); --- -++ struct FoldSqrt { -++ std::optional operator()(APFloat operand) { -++ if (operand.getSizeInBits(operand.getSemantics()) == 64) -++ return APFloat(std::sqrt(operand.convertToDouble())); -+ - - if (auto res = constFoldUnaryOp( - - lhsAttr, foldSqrt)) { - - rewriter.replaceOpWithNewOp( - - op, op.getType(), llvm::cast(res)); - - return success(); --- } --- -++ if (operand.getSizeInBits(operand.getSemantics()) == 32) -++ return APFloat(sqrtf(operand.convertToFloat())); -++ return std::nullopt; -+ } -+ - - return rewriter.notifyMatchFailure(op, "unable to fold sqrt"); - - } - - -@@ -1973,32 +2011,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - return APFloat(sqrtf(a.convertToFloat())); - - return {}; - - } ---}; --- ---struct EvalIotaOpPattern : public FoldOpRewritePattern { --+ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); --+ if (failed(res)) return failure(); --+ rewriter.replaceOpWithNewOp(op, res.value()); --+ return success(); --+ } --+ --+ struct FoldSqrt { --+ std::optional operator()(APFloat operand) { --+ if (operand.getSizeInBits(operand.getSemantics()) == 64) --+ return APFloat(std::sqrt(operand.convertToDouble())); --+ --+ if (operand.getSizeInBits(operand.getSemantics()) == 32) --+ return APFloat(sqrtf(operand.convertToFloat())); --+ return std::nullopt; --+ } --+ - + // TODO: Enable int folding. - + std::optional operator()(APInt operand) { - + return std::nullopt; - + } - + }; --+}; --+ -+ }; -+ -+-struct EvalIotaOpPattern : public FoldOpRewritePattern { - +struct FoldIotaOpPattern : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; - -@@ -2015,7 +2035,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - - auto elementType = resultType.getElementType(); - --@@ -929,7 +1052,7 @@ -+@@ -929,7 +1052,7 @@ DenseElementsAttr transposeType(TransposeOp& op, const RangeType& data) { - // transpose(constant) => constant with permuted dimensions - // This covers ranked tensor types with 0 dimensions(zero elements) and 0 - // rank(scalar), as well as splat values. -@@ -2024,7 +2044,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - using FoldOpRewritePattern::FoldOpRewritePattern; - - LogicalResult matchAndRewrite(TransposeOp op, --@@ -943,6 +1066,7 @@ -+@@ -943,6 +1066,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { - return rewriter.notifyMatchFailure( - op, "expected constant integer or float operand"); - -@@ -2032,7 +2052,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - DenseElementsAttr resAttr; - if (auto data = els.tryGetValues()) - resAttr = transposeType(op, *data); --@@ -957,6 +1081,7 @@ -+@@ -957,6 +1081,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { - } - }; - -@@ -2040,7 +2060,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - struct LowerBoolSplatConstantsIntoReduceOpRegion - : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; --@@ -1160,7 +1285,7 @@ -+@@ -1160,7 +1285,7 @@ bool hasNoDeclaredSideEffects(Operation* op) { - return true; - } - -@@ -2049,7 +2069,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - : public FoldOpRewritePattern { - using FoldOpRewritePattern::FoldOpRewritePattern; - --@@ -1233,23 +1358,16 @@ -+@@ -1233,23 +1358,16 @@ void populateStablehloAggressiveFolderPatterns( - PatternBenefit benefit) { - populateStablehloShapeFolderPatterns(context, patterns, options, benefit); - -@@ -2083,7 +2103,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - - class StablehloTargetIndependentOptimizationPass { --@@ -1266,25 +1384,25 @@ -+@@ -1266,25 +1384,25 @@ void populateStablehloShapeFolderPatterns( - MLIRContext* context, RewritePatternSet* patterns, - const StablehloAggressiveFolderPassOptions& options, - PatternBenefit benefit) { -@@ -2128,28 +2148,29 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold - } - - void populateStablehloShapeFolderPatterns(MLIRContext* context, --diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td ----- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td --+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td --@@ -48,6 +48,8 @@ -- def RankEqual : Constraint< -+diff --git a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -+index 8adb6dbc..3a8edc61 100644 -+--- a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -++++ b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td -+@@ -49,6 +49,8 @@ def RankEqual : Constraint< - CPred<"llvm::cast($0.getType()).getRank() == llvm::cast($1.getType()).getRank()">, - "same rank">; --+ --+def TensorDimsAllOne : Constraint, "all tensor dims are 1">; - -++def TensorDimsAllOne : Constraint, "all tensor dims are 1">; -++ - def TypesEqual : Constraint, "operands are equal">; - --@@ -100,6 +102,8 @@ -- def ZeroExtent : AttrConstraint< -+ /////////// -+@@ -101,6 +103,8 @@ def ZeroExtent : AttrConstraint< - CPred<"cast($_self).getNumElements() == 0">, - "is zero extent">; --+ --+def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; - -++def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; -++ - /////////// - //// Native Code Call Utilities --@@ -503,7 +507,7 @@ -+ -+@@ -503,7 +507,7 @@ def SelectOp_InvertBroadcastPredicateAndSwap - // Must be static shape, otherwise would require broadcasting via - // CHLO_ConstantLike. - def SubtractOp_FoldToZero -@@ -2158,4 +2179,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimp - (StableHLO_ConstantLike<"0"> $operand)>; - - // Pattern: subtract(X, 0) -> X -+-- -+2.48.1 - --- -2.48.1 - diff --git a/third_party/modules/xla/20250612.0-6e48cbb/source.json b/third_party/modules/xla/20250612.0-6e48cbb/source.json deleted file mode 100644 index 6a4e208b..00000000 --- a/third_party/modules/xla/20250612.0-6e48cbb/source.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "strip_prefix": "xla-6e48cbb8d33d771c964697e39bfaf678bcc6de31", - "url": "https://github.com/openxla/xla/archive/6e48cbb8d33d771c964697e39bfaf678bcc6de31.tar.gz", - "integrity": "sha256-i9lYvZ2MkzfyVW2Iu3qIucXIgGEhkbwsYXCrUZ6Yze8=", - "overlay": { - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" - } -} diff --git a/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel deleted file mode 100644 index 09bc2307..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel +++ /dev/null @@ -1,58 +0,0 @@ -module( - name = "xla", - version = "20250710.0-22ea002", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") -use_repo( - toolchains_private, - "rules_ml_toolchain", -) - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl deleted file mode 100644 index b8c79070..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _toolchains_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -toolchains_private = module_extension( - implementation = _toolchains_private_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl deleted file mode 100644 index 9dfe0b16..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl +++ /dev/null @@ -1,73 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - # Use cuda_configure from XLA to make it work with bzlmod. - # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. - # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: - # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. - # - Load: Then, it requires a load() statement to load that same file to continue the setup. - # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before - # the Analysis Phase (when repository rules are run). - # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. - # Without using the official WORKSPACE.bzlmod escape hatch, - # this incompatibility cannot be resolved without modifying the upstream rules. - - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", - strip_prefix = "grpc-1.68.2", - patch_file = [ - "//third_party/grpc:grpc.patch", - ], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", - strip_prefix = "protobuf-5.28.3", - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), - ) - - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch deleted file mode 100644 index 6df5b1b3..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Fri, 11 Jul 2025 14:05:16 +0200 -Subject: [PATCH] zml patch - ---- - third_party/stablehlo/workspace.bzl | 1 + - third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ - 2 files changed, 94 insertions(+) - create mode 100644 third_party/stablehlo/zml.patch - -diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl -index d9d5063744..44980948d0 100644 ---- a/third_party/stablehlo/workspace.bzl -+++ b/third_party/stablehlo/workspace.bzl -@@ -15,5 +15,6 @@ def repo(): - urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), - patch_file = [ - "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. -+ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. - ], - ) -diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch -new file mode 100644 -index 0000000000..2a09384582 ---- /dev/null -+++ b/third_party/stablehlo/zml.patch -@@ -0,0 +1,93 @@ -+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 -+From: Hugo Mano -+Date: Fri, 11 Jul 2025 12:08:35 +0200 -+Subject: [PATCH] zml patch -+ -+--- -+ stablehlo/dialect/Serialization.cpp | 5 ++--- -+ stablehlo/dialect/Serialization.h | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- -+ 5 files changed, 6 insertions(+), 9 deletions(-) -+ -+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp -+index cb89d673..4370d588 100644 -+--- a/stablehlo/dialect/Serialization.cpp -++++ b/stablehlo/dialect/Serialization.cpp -+@@ -39,8 +39,7 @@ namespace stablehlo { -+ -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects) { -++ raw_ostream& os) { -+ MLIRContext* context = module.getContext(); -+ -+ // Convert StableHLO --> VHLO. -+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, -+ { -+ PassManager pm(context); -+ StablehloLegalizeToVhloPassOptions options; -+- options.allowOtherDialects = allowOtherDialects; -++ options.allowOtherDialects = false; -+ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); -+ if (!succeeded(pm.run(module))) { -+ return failure(); -+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h -+index 811ca97b..abe95e63 100644 -+--- a/stablehlo/dialect/Serialization.h -++++ b/stablehlo/dialect/Serialization.h -+@@ -34,8 +34,7 @@ namespace stablehlo { -+ // unsupported dialects. -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects = false); -++ raw_ostream& os); -+ -+ // Read StableHLO portable artifact -+ // -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp -+index 343f8d0b..8f52e4d5 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp -++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp -+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( -+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -+ mlir::detail::CallbackOstream stream(callback, userData); -+ if (failed(mlir::stablehlo::serializePortableArtifact( -+- unwrap(moduleStr), unwrap(targetVersion), stream, -+- allowOtherDialects))) -++ unwrap(moduleStr), unwrap(targetVersion), stream))) -+ return mlirLogicalResultFailure(); -+ return mlirLogicalResultSuccess(); -+ } -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index fdf0d6a9..8d5c8752 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( -+ } -+ -+ return stablehlo::serializePortableArtifact( -+- module, targetVersion, os, allowOtherDialectsOption.getValue()); -++ module, targetVersion, os); -+ }, -+ [](DialectRegistry ®istry) { -+ mlir::registerAllDialects(registry); -+-- -+2.39.5 (Apple Git-154) -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250710.0-22ea002/source.json b/third_party/modules/xla/20250710.0-22ea002/source.json deleted file mode 100644 index d01c18a3..00000000 --- a/third_party/modules/xla/20250710.0-22ea002/source.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "strip_prefix": "xla-ef07e787ea1303fa2f8d8a175d24d434bfb84107", - "url": "https://github.com/openxla/xla/archive/ef07e787ea1303fa2f8d8a175d24d434bfb84107.tar.gz", - "integrity": "sha256-OL0e3Y+dgu/DM5MV+LsWvtOI9rUGLvXbwIU/9bnHGXc=", - "overlay": { - "toolchains_private.bzl": "", - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Remove-unconventional-C-code-in-headers.patch": "" - } -} diff --git a/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel deleted file mode 100644 index 6c1bde70..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel +++ /dev/null @@ -1,58 +0,0 @@ -module( - name = "xla", - version = "20250718.0-6319f0d", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.17") -bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.39.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") -bazel_dep(name = "rules_shell", version = "0.4.1") -bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") - -toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") -use_repo( - toolchains_private, - "rules_ml_toolchain", -) - -workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") -use_repo( - workspace_private, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", - "python_version_repo", - "tsl", -) - -workspace_public = use_extension("//:xla.bzl", "xla") -use_repo( - workspace_public, - "llvm-raw", - "stablehlo", - "triton", -) - -llvm = use_extension("//:llvm.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "AMDGPU", - "NVPTX", - "X86", - ], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl deleted file mode 100644 index b4a2fe4f..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl +++ /dev/null @@ -1,30 +0,0 @@ -load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list( - default = [], - ), - }, - ), - }, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl deleted file mode 100644 index b8c79070..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _toolchains_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -toolchains_private = module_extension( - implementation = _toolchains_private_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl deleted file mode 100644 index 9dfe0b16..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl +++ /dev/null @@ -1,73 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("//third_party/py:python_repo.bzl", "python_repository") -load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") -load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _workspace_private_impl(mctx): - http_archive( - name = "rules_ml_toolchain", - sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", - strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", - urls = [ - "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", - ], - ) - - # Use cuda_configure from XLA to make it work with bzlmod. - # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. - # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: - # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. - # - Load: Then, it requires a load() statement to load that same file to continue the setup. - # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before - # the Analysis Phase (when repository rules are run). - # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. - # Without using the official WORKSPACE.bzlmod escape hatch, - # this incompatibility cannot be resolved without modifying the upstream rules. - - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_vendored(name = "tsl", relpath = "third_party/tsl") - pybind11_bazel() - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", - strip_prefix = "grpc-1.68.2", - patch_file = [ - "//third_party/grpc:grpc.patch", - ], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["//third_party/protobuf:protobuf.patch"], - sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", - strip_prefix = "protobuf-5.28.3", - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), - ) - - python_repository( - name = "python_version_repo", - requirements_versions = ["3.11"], - requirements_locks = ["//:requirements_lock_3_11.txt"], - local_wheel_workspaces = [], - local_wheel_dist_folder = None, - default_python_version = None, - local_wheel_inclusion_list = ["*"], - local_wheel_exclusion_list = [], - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -workspace_private = module_extension( - implementation = _workspace_private_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl deleted file mode 100644 index f14bf2a9..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("//third_party/llvm:workspace.bzl", llvm = "repo") -load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") -load("//third_party/triton:workspace.bzl", triton = "repo") - -def _xla_impl(mctx): - triton() - llvm("llvm-raw") - stablehlo() - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla = module_extension( - implementation = _xla_impl, -) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch deleted file mode 100644 index 8924cf4b..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 -From: Jean-Baptiste Dalido -Date: Mon, 6 Jan 2025 13:33:13 +0100 -Subject: [PATCH] bazel: migration to bazel 8.0.1 - ---- - .bazelversion | 2 +- - third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/.bazelversion b/.bazelversion -index f22d756da3..fa5fce04b3 100644 ---- a/.bazelversion -+++ b/.bazelversion -@@ -1 +1 @@ --7.4.1 -+8.1.1 -\ No newline at end of file -diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl -index d62531152d..71d80a5a99 100644 ---- a/third_party/gpus/cuda_configure.bzl -+++ b/third_party/gpus/cuda_configure.bzl -@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. - load( - "@bazel_tools//tools/cpp:lib_cc_configure.bzl", - "escape_string", -- "get_env_var", - ) - load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", -- "find_msvc_tool", - "find_vc_path", - "setup_vc_env_vars", - ) -+load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") -+load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") - load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") - load( - "//third_party/remote_config:common.bzl", --- -2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch deleted file mode 100644 index 938ef401..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Tue, 27 May 2025 11:48:17 +0200 -Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt - -PR: https://github.com/openxla/xla/pull/13420 ---- - xla/pjrt/c/BUILD | 5 +++++ - xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ - xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- - 3 files changed, 57 insertions(+), 1 deletion(-) - -diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD -index 79f18fa0bc..0f33dd8a6e 100644 ---- a/xla/pjrt/c/BUILD -+++ b/xla/pjrt/c/BUILD -@@ -69,8 +69,13 @@ cc_library( - ":pjrt_c_api_wrapper_impl", - "//xla/ffi:execution_context", - "//xla/ffi:type_id_registry", -+ "//xla/ffi:ffi_api", -+ "//xla/ffi/api:c_api", -+ "//xla/ffi/api:ffi", -+ "//xla/service:custom_call_target_registry", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings:string_view", -+ "@com_google_absl//absl/strings:str_format", - ], - ) - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -index 995a2c7e50..b8f10bc2f7 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h -@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); - // Adds a user data to the execute context. - typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); - -+typedef enum PJRT_FFI_Handler_TraitsBits { -+ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, -+} PJRT_FFI_Handler_TraitsBits; -+ -+struct PJRT_FFI_Register_Handler_Args { -+ size_t struct_size; -+ const char* target_name; -+ size_t target_name_size; -+ int api_version; // 0 for an untyped call, 1 -- for typed -+ void* handler; -+ const char* platform_name; -+ size_t platform_name_size; -+ PJRT_FFI_Handler_TraitsBits traits; -+}; -+PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); -+ -+// Registers an FFI call handler for a specific platform. -+typedef PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args); -+ - typedef struct PJRT_FFI_Extension { - PJRT_Extension_Base base; - PJRT_FFI_TypeID_Register* type_id_register; - PJRT_FFI_UserData_Add* user_data_add; -+ PJRT_FFI_Register_Handler* register_handler; - } PJRT_FFI; - PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); - -diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -index 5fa88eab33..763270331b 100644 ---- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -+++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc -@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and - limitations under the License. - ==============================================================================*/ - --#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" -+#include - - #include "absl/status/status.h" -+#include "absl/strings/str_format.h" - #include "absl/strings/string_view.h" -+#include "xla/ffi/api/c_api.h" - #include "xla/ffi/execution_context.h" - #include "xla/ffi/type_id_registry.h" -+#include "xla/ffi/ffi_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" - #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" - #include "xla/pjrt/c/pjrt_c_api_helpers.h" - #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" -+#include "xla/service/custom_call_target_registry.h" - - namespace pjrt { - -@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { - return nullptr; - } - -+static PJRT_Error* PJRT_FFI_Register_Handler( -+ PJRT_FFI_Register_Handler_Args* args) { -+ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( -+ "PJRT_FFI_Register_Handler_Args", -+ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); -+ std::string target_name(args->target_name, args->target_name_size); -+ std::string platform_name(args->platform_name, args->platform_name_size); -+ switch (args->api_version) { -+ case 0: -+ xla::CustomCallTargetRegistry::Global()->Register( -+ target_name, args->handler, platform_name); -+ return nullptr; -+ case 1: -+ xla::ffi::Ffi::RegisterStaticHandler( -+ xla::ffi::GetXlaFfiApi(), target_name, platform_name, -+ reinterpret_cast(args->handler)); -+ return nullptr; -+ default: -+ return new PJRT_Error{absl::UnimplementedError( -+ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " -+ "Supported versions are 0 and 1.", -+ args->api_version))}; -+ } -+} -+ - PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - return { - PJRT_Extension_Base{ -@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { - }, - /*type_id_register=*/PJRT_FFI_TypeID_Register, - /*user_data_add=*/PJRT_FFI_UserData_Add, -+ /*register_handler=*/PJRT_FFI_Register_Handler, - }; - } - --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch deleted file mode 100644 index 6df5b1b3..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch +++ /dev/null @@ -1,124 +0,0 @@ -From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 -From: Hugo Mano -Date: Fri, 11 Jul 2025 14:05:16 +0200 -Subject: [PATCH] zml patch - ---- - third_party/stablehlo/workspace.bzl | 1 + - third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ - 2 files changed, 94 insertions(+) - create mode 100644 third_party/stablehlo/zml.patch - -diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl -index d9d5063744..44980948d0 100644 ---- a/third_party/stablehlo/workspace.bzl -+++ b/third_party/stablehlo/workspace.bzl -@@ -15,5 +15,6 @@ def repo(): - urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), - patch_file = [ - "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. -+ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. - ], - ) -diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch -new file mode 100644 -index 0000000000..2a09384582 ---- /dev/null -+++ b/third_party/stablehlo/zml.patch -@@ -0,0 +1,93 @@ -+From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 -+From: Hugo Mano -+Date: Fri, 11 Jul 2025 12:08:35 +0200 -+Subject: [PATCH] zml patch -+ -+--- -+ stablehlo/dialect/Serialization.cpp | 5 ++--- -+ stablehlo/dialect/Serialization.h | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- -+ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- -+ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- -+ 5 files changed, 6 insertions(+), 9 deletions(-) -+ -+diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp -+index cb89d673..4370d588 100644 -+--- a/stablehlo/dialect/Serialization.cpp -++++ b/stablehlo/dialect/Serialization.cpp -+@@ -39,8 +39,7 @@ namespace stablehlo { -+ -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects) { -++ raw_ostream& os) { -+ MLIRContext* context = module.getContext(); -+ -+ // Convert StableHLO --> VHLO. -+@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, -+ { -+ PassManager pm(context); -+ StablehloLegalizeToVhloPassOptions options; -+- options.allowOtherDialects = allowOtherDialects; -++ options.allowOtherDialects = false; -+ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); -+ if (!succeeded(pm.run(module))) { -+ return failure(); -+diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h -+index 811ca97b..abe95e63 100644 -+--- a/stablehlo/dialect/Serialization.h -++++ b/stablehlo/dialect/Serialization.h -+@@ -34,8 +34,7 @@ namespace stablehlo { -+ // unsupported dialects. -+ LogicalResult serializePortableArtifact(ModuleOp module, -+ StringRef targetVersion, -+- raw_ostream& os, -+- bool allowOtherDialects = false); -++ raw_ostream& os); -+ -+ // Read StableHLO portable artifact -+ // -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp -+index 343f8d0b..8f52e4d5 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.cpp -++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp -+@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( -+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { -+ mlir::detail::CallbackOstream stream(callback, userData); -+ if (failed(mlir::stablehlo::serializePortableArtifact( -+- unwrap(moduleStr), unwrap(targetVersion), stream, -+- allowOtherDialects))) -++ unwrap(moduleStr), unwrap(targetVersion), stream))) -+ return mlirLogicalResultFailure(); -+ return mlirLogicalResultSuccess(); -+ } -+diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h -+index 385156bf..24d11c1d 100644 -+--- a/stablehlo/integrations/c/StablehloDialectApi.h -++++ b/stablehlo/integrations/c/StablehloDialectApi.h -+@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, -+ MlirStringRef targetVersion, -+ MlirStringCallback callback, -+ void* userData, -+- bool allowOtherDialects = false); -++ bool allowOtherDialects); -+ -+ // Read a StableHLO program from a portable artifact, returning the module as -+ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, -+diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp -+index fdf0d6a9..8d5c8752 100644 -+--- a/stablehlo/tools/StablehloTranslateMain.cpp -++++ b/stablehlo/tools/StablehloTranslateMain.cpp -+@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( -+ } -+ -+ return stablehlo::serializePortableArtifact( -+- module, targetVersion, os, allowOtherDialectsOption.getValue()); -++ module, targetVersion, os); -+ }, -+ [](DialectRegistry ®istry) { -+ mlir::registerAllDialects(registry); -+-- -+2.39.5 (Apple Git-154) -+ --- -2.39.5 (Apple Git-154) - diff --git a/third_party/modules/xla/20250718.0-6319f0d/source.json b/third_party/modules/xla/20250718.0-6319f0d/source.json deleted file mode 100644 index a34f9a9e..00000000 --- a/third_party/modules/xla/20250718.0-6319f0d/source.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "strip_prefix": "xla-6319f0d3bdfd3078e04bb984a759c890b7116484", - "url": "https://github.com/openxla/xla/archive/6319f0d3bdfd3078e04bb984a759c890b7116484.tar.gz", - "integrity": "sha256-0XAMbJ4tn5yFllPGVCSh8st7lgTOUoyeypoudCB8tsY=", - "overlay": { - "toolchains_private.bzl": "", - "llvm.bzl": "", - "MODULE.bazel": "", - "workspace_private.bzl": "", - "xla.bzl": "" - }, - "patch_strip": 1, - "patches": { - "0001-bazel-migration-to-bazel-8.1.1.patch": "", - "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", - "0003-Remove-unconventional-C-code-in-headers.patch": "" - } -} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json deleted file mode 100644 index 4fa86cba..00000000 --- a/third_party/modules/xla/metadata.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "homepage": "https://github.com/openxla/xla", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/xla" - ], - "versions": [ - "20240902.0-d18cd64", - "20240919.0-1b18dd6", - "20241025.0-4663f04", - "20250103.0-5f1fe6a", - "20250122.0-cc075be", - "20250123.0-cc075be", - "20250204-0-6789523", - "20250317.0-71c67e2", - "20250317.1-71c67e2", - "20250317.2-71c67e2", - "20250527.0-cb67f2f", - "20250612.0-6e48cbb", - "20250710.0-22ea002", - "20250718.0-6319f0d" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel deleted file mode 100644 index 4fb39921..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20240722.0-c644d11", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch b/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch deleted file mode 100644 index 67353922..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/bootstrapped-generator/main.zig -+++ b/bootstrapped-generator/main.zig -@@ -6,7 +6,7 @@ const descriptor = @import("google/protobuf/descriptor.pb.zig"); - const mem = std.mem; - const FullName = @import("./FullName.zig").FullName; - --const USE_MODULES = false; -+const USE_MODULES = true; - - const allocator = std.heap.page_allocator; - diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json b/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json deleted file mode 100644 index f985b249..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-c644d1105caaca4d9a48d3536e253687052f74c8", - "url": "https://github.com/gwenzek/zig-protobuf/archive/c644d1105caaca4d9a48d3536e253687052f74c8.tar.gz", - "integrity": "sha256-KoZp8MLLEQQgS7Uip4sdjw7TYmhzKPdpilQYKp5iYV4=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-USE_MODULES-true.patch": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel deleted file mode 100644 index 0b9ec156..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250213.0-5304067", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/source.json b/third_party/modules/zig-protobuf/20250213.0-5304067/source.json deleted file mode 100644 index e4395bdd..00000000 --- a/third_party/modules/zig-protobuf/20250213.0-5304067/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-5304067205135532c0ad57f78243bfe86dc1ad3f", - "url": "https://github.com/gwenzek/zig-protobuf/archive/5304067205135532c0ad57f78243bfe86dc1ad3f.tar.gz", - "integrity": "sha256-cdby+J1CGNOvLRvdeASPNlB7y1q59LYcgjFOooBvIcA=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel deleted file mode 100644 index 40830df4..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250318.0-930153e", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/source.json b/third_party/modules/zig-protobuf/20250318.0-930153e/source.json deleted file mode 100644 index fa33ec5e..00000000 --- a/third_party/modules/zig-protobuf/20250318.0-930153e/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-930153ec955853ff9fcf8a55ba632b2b8051f731", - "url": "https://github.com/zml/zig-protobuf/archive/930153ec955853ff9fcf8a55ba632b2b8051f731.tar.gz", - "integrity": "sha256-ORRDxK/cLQ2E+c9l2lcj8FgIUHH+uEk/+qWVWuHZMEY=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel deleted file mode 100644 index b71798cc..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250716.0-97f1e31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel deleted file mode 100644 index b71798cc..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20250716.0-97f1e31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20250613.0-567662a") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json deleted file mode 100644 index 4975cdb8..00000000 --- a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-97f1e314d9bbf600fbaadee5bb08c8856e0006de", - "url": "https://github.com/zml/zig-protobuf/archive/97f1e314d9bbf600fbaadee5bb08c8856e0006de.tar.gz", - "integrity": "sha256-iAqsLIKhl15JInJvwMY8NGVll6eUBXzdLW6Xv8wANGY=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json deleted file mode 100644 index 4a038360..00000000 --- a/third_party/modules/zig-protobuf/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-protobuf", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "zml/zig-protobuf" - ], - "versions": [ - "20240722.0-c644d11", - "20250213.0-5304067", - "20250318.0-930153e", - "20250716.0-97f1e31", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel deleted file mode 100644 index 151f3337..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "zig-yaml", - version = "20240903.0-83d5fdf", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel deleted file mode 100644 index 94897f78..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "zig-yaml", - srcs = glob([ - "src/*.zig", - "src/parse/*.zig", - "src/yaml/*.zig", - ]), - main = "src/yaml.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json b/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json deleted file mode 100644 index 643aa238..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-yaml-83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b", - "url": "https://github.com/gwenzek/zig-yaml/archive/83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b.tar.gz", - "integrity": "sha256-uusblAIZQgyKlg0qVmuHWD2JG+dMM+N6oKNGZsBdpEk=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-yaml/metadata.json b/third_party/modules/zig-yaml/metadata.json deleted file mode 100644 index 3aa2c43c..00000000 --- a/third_party/modules/zig-yaml/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-yaml", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:gwenzek/zig-yaml" - ], - "versions": [ - "20240903.0-83d5fdf" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel deleted file mode 100644 index a59b2d87..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zigcoro", - version = "20240829.0-fc1db29", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "libxev", version = "20240825.0-dbe2291") diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel deleted file mode 100644 index 2aa934ca..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -genrule( - name = "libcoro_options_gen", - outs = ["options.zig"], - cmd = """\ -cat < $@ -pub const default_stack_size: usize = 4096; -pub const debug_log_level: usize = 0; -EOF -""", -) - -zig_library( - name = "libcoro_options", - main = ":libcoro_options_gen", -) - -zig_library( - name = "libcoro", - srcs = glob(["src/*.zig"]), - extra_srcs = glob(["src/asm/*.s"]), - main = "src/main.zig", - visibility = ["//visibility:public"], - deps = [ - ":libcoro_options", - "@libxev", - ], -) diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch deleted file mode 100644 index 6072de07..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch +++ /dev/null @@ -1,48 +0,0 @@ -From e2416ff5ec9cad5572e6a5edac3f949b75024c00 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 11:34:35 +0200 -Subject: [PATCH 1/2] Handle generic function in Signature - ---- - src/coro.zig | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/src/coro.zig b/src/coro.zig -index 9285489..8a34717 100644 ---- a/src/coro.zig -+++ b/src/coro.zig -@@ -266,6 +266,7 @@ const CoroT = struct { - YieldT: type = void, - InjectT: type = void, - ArgsT: type, -+ _ReturnT: type, - - // If the function this signature represents is compile-time known, - // it can be held here. -@@ -273,19 +274,21 @@ const CoroT = struct { - - fn init(comptime Func: anytype, comptime options: CoroT.Options) @This() { - const FuncT = if (@TypeOf(Func) == type) Func else @TypeOf(Func); -+ const ArgsT = options.ArgsT orelse ArgsTuple(FuncT); - return .{ - .Func = FuncT, - .YieldT = options.YieldT, - .InjectT = options.InjectT, -- .ArgsT = options.ArgsT orelse ArgsTuple(FuncT), -+ .ArgsT = ArgsT, - .func_ptr = if (@TypeOf(Func) == type) null else struct { - const val = Func; - }, -+ ._ReturnT = @TypeOf(@call(.auto, Func, @as(ArgsT, undefined))), - }; - } - - pub fn ReturnT(comptime self: @This()) type { -- return @typeInfo(self.Func).Fn.return_type.?; -+ return self._ReturnT; - } - }; - --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch deleted file mode 100644 index f1e68622..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d8a4773ee9fa0ae5c5fd91e84c322e75a634a077 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 13:14:59 +0200 -Subject: [PATCH 2/2] asyncio: return error super set in pread/pwrite - ---- - src/asyncio.zig | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/asyncio.zig b/src/asyncio.zig -index 68cc7d3..8487754 100644 ---- a/src/asyncio.zig -+++ b/src/asyncio.zig -@@ -354,7 +354,7 @@ pub const File = struct { - } - - const PReadResult = xev.ReadError!usize; -- pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) PReadResult { -+ pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) !usize { - const ResultT = PReadResult; - const Data = struct { - result: ResultT = undefined, -@@ -391,7 +391,7 @@ pub const File = struct { - } - - const PWriteResult = xev.WriteError!usize; -- pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) PWriteResult { -+ pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) !usize { - const ResultT = PWriteResult; - const Data = struct { - result: ResultT = undefined, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json b/third_party/modules/zigcoro/20240829.0-fc1db29/source.json deleted file mode 100644 index 17e66d4e..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "zigcoro-fc1db29dee049ce572a516f1586cce0eca4fcb2a", - "url": "https://github.com/rsepassi/zigcoro/archive/fc1db29dee049ce572a516f1586cce0eca4fcb2a.tar.gz", - "integrity": "sha256-2ynCPLSRFPto3HIOhDTBDyUS770PNMI9ioMkj8Z7Y+s=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Handle-generic-function-in-Signature.patch": "", - "0002-asyncio-return-error-super-set-in-pread-pwrite.patch": "" - } -} diff --git a/third_party/modules/zigcoro/metadata.json b/third_party/modules/zigcoro/metadata.json deleted file mode 100644 index 533957ce..00000000 --- a/third_party/modules/zigcoro/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/rsepassi/zigcoro", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:rsepassi/zigcoro" - ], - "versions": [ - "20240829.0-fc1db29" - ], - "yanked_versions": {} -} diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index fbeee624..876b0db2 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,26 +1,46 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") +load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") +load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") +load("//third_party/cute_ir:repo.bzl", cute_ir = "repo") +load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") +load("//third_party/flydsl:repo.bzl", flydsl = "repo") +load("//third_party/iree:repo.bzl", iree = "repo") +load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") +load("//third_party/linenoise:repo.bzl", linenoise = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") +load("//third_party/stb:repo.bzl", stb = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") +load("//third_party/uucode:repo.bzl", uucode = "repo") load("//third_party/xla:repo.bzl", xla = "repo") -load("//third_party/flashattn:repo.bzl", flashattn = "repo") -load("//third_party/linenoise:repo.bzl", linenoise = "repo") +load("//third_party/zigimg:repo.bzl", zigimg = "repo") +load("//third_party/zio:repo.bzl", zio = "repo") +load("//third_party/zls:repo.bzl", zls = "repo") def _non_module_deps_impl(mctx): + cloud_accelerator_diagnostics() com_google_sentencepiece() org_swig_swig() mnist() xla() arocc() translate_c() - flashattn() + flashinfer_cutlass_moe() linenoise() + stb() + zigimg() + uucode() + libvaxis() + iree() + cuda_tile() + flydsl() + cute_ir() + zio() + zls() return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], ) non_module_deps = module_extension( diff --git a/third_party/org_swig_swig/repo.bzl b/third_party/org_swig_swig/repo.bzl index c35659bf..9a6c55bb 100644 --- a/third_party/org_swig_swig/repo.bzl +++ b/third_party/org_swig_swig/repo.bzl @@ -3,7 +3,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): http_archive( name = "org_swig_swig", - url = "http://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + urls = [ + "https://downloads.sourceforge.net/project/swig/swig/swig-4.4.1/swig-4.4.1.tar.gz", + "https://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + ], sha256 = "40162a706c56f7592d08fd52ef5511cb7ac191f3593cf07306a0a554c6281fcf", strip_prefix = "swig-4.4.1", build_file = "//:third_party/org_swig_swig/swig.bazel", diff --git a/third_party/rules_distroless/accept-nospace-version-constraints.patch b/third_party/rules_distroless/accept-nospace-version-constraints.patch new file mode 100644 index 00000000..62973749 --- /dev/null +++ b/third_party/rules_distroless/accept-nospace-version-constraints.patch @@ -0,0 +1,28 @@ +diff --git a/apt/private/version_constraint.bzl b/apt/private/version_constraint.bzl +index 531fa01..b3a3d30 100644 +--- a/apt/private/version_constraint.bzl ++++ b/apt/private/version_constraint.bzl +@@ -2,11 +2,19 @@ + + load(":version.bzl", version_lib = "version") + ++_VERSION_CONSTRAINT_OPERATORS = ["<<", ">>", "<=", ">=", "="] ++ + def _parse_version_constraint(rawv): +- vconst_i = rawv.find(" ") +- if vconst_i == -1: +- fail('invalid version string %s expected a version constraint ">=", "=", ">=", "<<", ">>"' % rawv) +- return (rawv[:vconst_i], rawv[vconst_i + 1:]) ++ rawv = rawv.strip() ++ for op in _VERSION_CONSTRAINT_OPERATORS: ++ if rawv.startswith(op): ++ version = rawv[len(op):].strip() ++ if version: ++ return (op, version) ++ ++ break ++ ++ fail('invalid version string %s expected a version constraint ">=", "=", "<=", "<<", ">>"' % rawv) + + def _parse_dep(raw): + raw = raw.strip() # remove leading & trailing whitespace diff --git a/third_party/rules_distroless/flat-repositories.patch b/third_party/rules_distroless/flat-repositories.patch new file mode 100644 index 00000000..e4ba73bc --- /dev/null +++ b/third_party/rules_distroless/flat-repositories.patch @@ -0,0 +1,59 @@ +diff --git a/apt/private/deb_resolve.bzl b/apt/private/deb_resolve.bzl +--- a/apt/private/deb_resolve.bzl ++++ b/apt/private/deb_resolve.bzl +@@ -46,7 +46,10 @@ + sources = [] + + for src in manifest["sources"]: +- distr, components = src["channel"].split(" ", 1) ++ if src["channel"].endswith("/"): ++ distr, components = src["channel"], "" ++ else: ++ distr, components = src["channel"].split(" ", 1) + for comp in components.split(" "): + # TODO: only support urls before 1.0 + if "urls" in src: +@@ -60,6 +63,7 @@ + urls, + distr, + comp, ++ src.get("archs", manifest["archs"]), + )) + + repository = deb_repository.new(rctx, sources = sources, archs = manifest["archs"]) +diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl +--- a/apt/private/apt_deb_repository.bzl ++++ b/apt/private/apt_deb_repository.bzl +@@ -13,7 +13,7 @@ + return use_netrc(netrc, urls, {}) + + def _fetch_package_index(rctx, urls, dist, comp, arch, integrity): +- target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch) ++ target_triple = "{dist}/{comp}/{arch}".format(dist = dist.strip("/") or ".", comp = comp, arch = arch) + + # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2 + # --keep -> keep the original file (Bazel might be still committing the output to the cache) +@@ -35,7 +35,10 @@ + download = None + for (ext, cmd) in supported_extensions: + output = "{}/Packages{}".format(target_triple, ext) +- dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) ++ if dist.endswith("/"): ++ dist_url = "{}/{}Packages{}".format(url, dist.lstrip("/"), ext) ++ else: ++ dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) + auth = {} + if url in base_auth: + auth = {dist_url: base_auth[url]} +@@ -167,7 +170,10 @@ + ) + + for arch in archs: +- for (urls, dist, comp) in sources: ++ for (urls, dist, comp, source_archs) in sources: ++ if arch not in source_archs: ++ continue ++ + # We assume that `url` does not contain a trailing forward slash when passing to + # functions below. If one is present, remove it. Some HTTP servers do not handle + # redirects properly when a path contains "//" diff --git a/third_party/stb/BUILD.bazel b/third_party/stb/BUILD.bazel new file mode 100644 index 00000000..f91653e8 --- /dev/null +++ b/third_party/stb/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["stb.BUILD"]) diff --git a/third_party/stb/repo.bzl b/third_party/stb/repo.bzl new file mode 100644 index 00000000..44645c46 --- /dev/null +++ b/third_party/stb/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "stb", + remote = "https://github.com/nothings/stb.git", + build_file = "//third_party/stb:stb.BUILD", + commit = "f1c79c02822848a9bed4315b12c8c8f3761e1296", + ) diff --git a/third_party/stb/stb.BUILD b/third_party/stb/stb.BUILD new file mode 100644 index 00000000..b59ef43f --- /dev/null +++ b/third_party/stb/stb.BUILD @@ -0,0 +1,35 @@ +cc_import( + name = "stb_image_write", + hdrs = ["stb_image_write.h"], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "impl", + outs = ["impl.c"], + cmd = """\ +cat < $(@) +#include "stb_image.h" +#include "stb_image_resize2.h" +#include "stb_image_write.h" +EOF +""", +) + +cc_library( + name = "stb", + local_defines = [ + "STB_IMAGE_IMPLEMENTATION", + "STB_IMAGE_RESIZE_IMPLEMENTATION", + "STB_IMAGE_WRITE_IMPLEMENTATION", + ], + srcs = [":impl"], + hdrs = [ + "stb_image.h", + "stb_image_resize2.h", + "stb_image_write.h", + ], + includes = ["."], + visibility = ["//visibility:public"], +) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index ab5e9860..dfd8835d 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -1,9 +1,11 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") def repo(): - new_git_repository( + git_repository( name = "translate-c", - remote = "https://codeberg.org/ziglang/translate-c.git", - commit = "d2be2f19ef7c9caa1561d38c96581ac79dd4c654", - build_file = "//third_party/translate-c:translate-c.bazel", + # Use the ZML GitHub mirror because Codeberg is unreliable. + remote = "https://github.com/zml/translate-c", + # 2.0.0 + commit = "0da7a16c3235b935b82421646076e0657cda21f6", + build_file = Label("//third_party/translate-c:translate-c.bazel"), ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index f71603e3..7de2a5eb 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,4 +1,12 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option(name = "debug_allocations", bool = False) + +zig_options( + name = "build_options", + settings = [":debug_allocations"], +) zig_library( name = "c_builtins", @@ -14,21 +22,28 @@ zig_library( zig_binary( name = "translate-c_raw", + data = glob(["lib/**/*.zig"]), srcs = glob(["src/**/*.zig"]), main = "src/main.zig", linkopts = ["-lc"], - visibility = ["//visibility:public"], deps = [ + ":build_options", ":c_builtins", ":helpers", "@arocc//:aro", ], + visibility = ["//visibility:public"], ) +# TODO(cerisier): Generate 1 translate-c toolchain per target platform +# So that the choice of doing this for a given platform is given to users. +# +# Right now we do this because the arocc source code has the same support +# for platforms than the official zig compiler and in the case of using a +# custom zig version with support for newer chips, arocc cannot be built. zig_configure_binary( name = "translate-c", actual = ":translate-c_raw", - no_translate_c = 1, - use_cc_common_link = 0, + bootstrapped = 0, visibility = ["//visibility:public"], ) diff --git a/third_party/uucode/BUILD.bazel b/third_party/uucode/BUILD.bazel new file mode 100644 index 00000000..49e3a399 --- /dev/null +++ b/third_party/uucode/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "build_config.zig", + "uucode.bazel", +]) diff --git a/third_party/uucode/build_config.zig b/third_party/uucode/build_config.zig new file mode 100644 index 00000000..a4e76bff --- /dev/null +++ b/third_party/uucode/build_config.zig @@ -0,0 +1,16 @@ +const config = @import("config.zig"); + +pub const fields = config.fields; +pub const build_components = config.build_components; +pub const get_components = config.get_components; + +pub const tables: []const config.Table = &.{ + .{ + .fields = &.{ + "east_asian_width", + "grapheme_break", + "general_category", + "is_emoji_presentation", + }, + }, +}; diff --git a/third_party/uucode/config-storage-module.patch b/third_party/uucode/config-storage-module.patch new file mode 100644 index 00000000..d390986c --- /dev/null +++ b/third_party/uucode/config-storage-module.patch @@ -0,0 +1,33 @@ +diff --git a/src/config.zig b/src/config.zig +index 2cadab4..aa46c5d 100644 +--- a/src/config.zig ++++ b/src/config.zig +@@ -1,5 +1,5 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); ++pub const storage = @import("storage.zig"); + const multi_slice = @import("multi_slice.zig"); + pub const quirks = @import("quirks.zig"); + pub const components = @import("components.zig"); +diff --git a/src/generate.zig b/src/generate.zig +index 6f4dcf5..8ef1d83 100644 +--- a/src/generate.zig ++++ b/src/generate.zig +@@ -1,7 +1,7 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); + const config = @import("config.zig"); + const build_config = @import("build_config"); ++const storage = config.storage; + const inlineAssert = config.quirks.inlineAssert; + + pub const std_options: std.Options = .{ +@@ -339,7 +339,7 @@ pub fn main(init: std.process.Init.Minimal) !void { + \\const std = @import("std"); + \\const config = @import("config.zig"); + \\const types = config.types; +- \\const storage = @import("storage.zig"); ++ \\const storage = config.storage; + \\const build_config = @import("build_config"); + \\ + \\pub const get_components = build_config.get_components; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl new file mode 100644 index 00000000..5f7b1cbf --- /dev/null +++ b/third_party/uucode/repo.bzl @@ -0,0 +1,29 @@ +def _uucode_repo_impl(ctx): + ctx.download_and_extract( + url = "https://github.com/jacobsandlund/uucode/archive/{}.tar.gz".format(ctx.attr.commit), + stripPrefix = "uucode-{}".format(ctx.attr.commit), + ) + ctx.patch(ctx.attr.config_storage_patch, strip = 1) + + ctx.symlink(ctx.attr.build_config, "build_config.zig") + ctx.symlink(ctx.attr.build_file, "BUILD.bazel") + +_uucode_repo = repository_rule( + implementation = _uucode_repo_impl, + attrs = { + "commit": attr.string(mandatory = True), + "build_config": attr.label(mandatory = True, allow_single_file = True), + "build_file": attr.label(mandatory = True, allow_single_file = True), + "config_storage_patch": attr.label(mandatory = True, allow_single_file = True), + }, +) + +def repo(): + _uucode_repo( + name = "uucode", + # main branch + commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", + build_config = "//third_party/uucode:build_config.zig", + build_file = "//third_party/uucode:uucode.bazel", + config_storage_patch = "//third_party/uucode:config-storage-module.patch", + ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel new file mode 100644 index 00000000..4a3710f2 --- /dev/null +++ b/third_party/uucode/uucode.bazel @@ -0,0 +1,91 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# Generator + +zig_library( + name = "_config_gen", + import_name = "config.zig", + main = "src/config.zig", + srcs = [ + "src/components.zig", + "src/ucd_inputs.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", + "src/storage.zig", + "src/types.zig", + ], +) + +zig_library( + name = "_build_config_gen", + import_name = "build_config", + main = "build_config.zig", + deps = [":_config_gen"], +) + +zig_binary( + name = "gen_tables", + main = "src/generate.zig", + deps = [":_build_config_gen", ":_config_gen"], +) + +genrule( + name = "tables", + srcs = glob(["ucd/**"]), + outs = ["tables.zig"], + cmd = "OUT=$$PWD/$@ && TOOL=$$PWD/$(execpath :gen_tables) && cd $$(dirname $(location ucd/UnicodeData.txt))/.. && $$TOOL $$OUT", + tools = [":gen_tables"], +) + +# Library + +zig_library( + name = "types", + import_name = "types.zig", + main = "src/types.zig", +) + +zig_library( + name = "config", + import_name = "config.zig", + main = "src/config.zig", + srcs = [ + "src/components.zig", + "src/ucd_inputs.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", + "src/storage.zig", + ], + deps = [":types"], +) + +zig_library( + name = "build_config", + import_name = "build_config", + main = "build_config.zig", + deps = [":config"], +) + +zig_library( + name = "tables_lib", + import_name = "tables", + main = ":tables", + deps = [":build_config", ":config"], +) + +zig_library( + name = "uucode", + import_name = "uucode", + main = "src/root.zig", + srcs = [ + "src/ascii.zig", + "src/code_point.zig", + "src/get.zig", + "src/grapheme.zig", + "src/utf8.zig", + ], + deps = [":config", ":tables_lib", ":types"], + visibility = ["//visibility:public"], +) diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel index e69de29b..75f731ff 100644 --- a/third_party/xla/BUILD.bazel +++ b/third_party/xla/BUILD.bazel @@ -0,0 +1 @@ +# Package marker for XLA patches diff --git a/third_party/xla/cuda-root-path-local-defines.patch b/third_party/xla/cuda-root-path-local-defines.patch new file mode 100644 index 00000000..c9ca6aa8 --- /dev/null +++ b/third_party/xla/cuda-root-path-local-defines.patch @@ -0,0 +1,25 @@ +diff --git a/xla/tsl/platform/default/BUILD b/xla/tsl/platform/default/BUILD +index f01bdfc67e..18a30d44a 100644 +--- a/xla/tsl/platform/default/BUILD ++++ b/xla/tsl/platform/default/BUILD +@@ -82,13 +82,13 @@ cc_library( + if_false = ["@cuda_nvcc//:nvvm"], + if_true = ["@cuda_nvvm//:nvvm"], + )), +- local_defines = [ +- "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", +- "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", +- "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", +- "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", +- "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", +- ], ++ # local_defines = [ ++ # "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", ++ # "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", ++ # "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", ++ # "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", ++ # "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", ++ # ], + tags = [ + "manual", + "no_oss", diff --git a/third_party/xla/llvm_raw_only.bzl b/third_party/xla/llvm_raw_only.bzl new file mode 100644 index 00000000..f2758994 --- /dev/null +++ b/third_party/xla/llvm_raw_only.bzl @@ -0,0 +1,13 @@ +load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") + +def _llvm_raw_only_impl(mctx): + llvm("llvm-raw") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["llvm-raw"], + root_module_direct_dev_deps = [], + ) + +llvm_raw_only = module_extension( + implementation = _llvm_raw_only_impl, +) diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 67740474..0e1d9620 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,5 +4,9 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "01da52b9afe3a2e694bd926323a649a6e63a3785", + commit = "65aed3e0ed7b0f965825676dd27e312965c846c8", + patches = [ + "//third_party/xla:cuda-root-path-local-defines.patch", + ], + patch_args = ["-p1"], ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 43a6629b..6634fa15 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -1,5 +1,11 @@ load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("@xla//third_party/eigen3:workspace.bzl", eigen3 = "repo") +load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("@xla//third_party/highwayhash:workspace.bzl", highwayhash = "repo") +load("@xla//third_party/hwloc:workspace.bzl", hwloc = "repo") load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") +load("@xla//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") +load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") load("@xla//third_party/triton:workspace.bzl", triton = "repo") @@ -24,7 +30,39 @@ simple_files = repository_rule( def _dummy_repos(mctx): simple_files(name = "local_config_cuda", files = { - "cuda/BUILD.bazel": "", + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +config_setting( + name = "is_cuda_enabled", + define_values = {"local_config_cuda_enabled": "true"}, +) + +config_setting( + name = "is_cuda_compiler_nvcc", + define_values = {"local_config_cuda_compiler_nvcc": "true"}, +) +""", + "cuda/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_config_cuda", + define_values = {"local_config_cuda_using_config": "true"}, +) + +config_setting( + name = "FALSE", + define_values = {"local_config_cuda_false": "true"}, +) +""", "cuda/build_defs.bzl": _BZL_HELPERS + """\ cuda_library = always_false if_cuda = always_if_false @@ -34,7 +72,22 @@ is_cuda_configured = always_false """, }) simple_files(name = "local_config_rocm", files = { - "rocm/BUILD.bazel": "", + "BUILD.bazel": "", + "rocm/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_hipcc", + define_values = {"local_config_rocm_using_hipcc": "true"}, +) +""", "rocm/build_defs.bzl": _BZL_HELPERS + """\ if_rocm = always_if_false if_rocm_is_configured = always_if_false @@ -47,23 +100,91 @@ get_rbe_amdgpu_pool = always_false }) simple_files(name = "local_config_sycl", files = { "BUILD.bazel": "", - "sycl/BUILD.bazel": "", + "sycl/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", "crosstool/BUILD.bazel": "", - "sycl/build_defs.bzl": _BZL_HELPERS + """\ + "sycl/build_defs.bzl": """\ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +""" + _BZL_HELPERS + """\ if_sycl = always_if_false if_sycl_is_configured = always_if_false +sycl_library = cc_library """, }) simple_files(name = "local_config_remote_execution", files = { "remote_execution.bzl": """gpu_test_tags = lambda: []""", }) simple_files(name = "local_config_tensorrt", files = { + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""", }) simple_files(name = "python_version_repo", files = { "py_version.bzl": """USE_PYWRAP_RULES = False""", }) simple_files(name = "rules_ml_toolchain", files = { + "py/rules_pywrap/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "pywrap.default.bzl", + "pywrap.impl.bzl", +]) + +filegroup( + name = "pywrap_bzl", + srcs = [ + "pywrap.default.bzl", + "pywrap.impl.bzl", + ], +) +""", + "py/rules_pywrap/pywrap.default.bzl": """\ +def use_pywrap_rules(): + return False + +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +pybind_extension = _unsupported +pywrap_aware_cc_import = _unsupported +pywrap_aware_filegroup = _unsupported +pywrap_aware_genrule = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", + "py/rules_pywrap/pywrap.impl.bzl": """\ +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +collected_pywrap_infos = _unsupported +generated_common_win_def_file = _unsupported +pybind_extension = _unsupported +python_extension = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", "third_party/gpus/BUILD.bazel": "", "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", "third_party/extensions/sycl_configure.bzl": "", @@ -74,17 +195,30 @@ if_sycl_is_configured = always_if_false def _xla_impl(mctx): llvm("llvm-raw") stablehlo() + shardy() triton() + farmhash() + highwayhash() + hwloc() + eigen3() + ml_dtypes() tf_http_archive( name = "com_github_grpc_grpc", - sha256 = "dd6a2fa311ba8441bbefd2764c55b99136ff10f7ea42954be96006a2723d33fc", - strip_prefix = "grpc-1.74.0", + sha256 = "41b695614b26652ff9e97ce50cfd4a6c7a3d45a9fe598d1454407746499bbf2c", + strip_prefix = "grpc-1.81.0", patch_file = ["//third_party/grpc:grpc.patch"], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.74.0.tar.gz"), + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.81.0.tar.gz"), ) tf_vendored(name = "tsl", path = "third_party/tsl") + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc", + strip_prefix = "snappy-1.2.2", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.2.tar.gz"), + ) _dummy_repos(mctx) return mctx.extension_metadata( diff --git a/third_party/zigimg/BUILD.bazel b/third_party/zigimg/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/zigimg/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl new file mode 100644 index 00000000..a4bb178f --- /dev/null +++ b/third_party/zigimg/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "zigimg", + remote = "https://github.com/zigimg/zigimg.git", + # Match libvaxis build.zig.zon. + commit = "734fa919fb15b7917d4eb124f53600348f4ffa30", + build_file = Label("//third_party/zigimg:zigimg.bazel"), + ) diff --git a/third_party/zigimg/zigimg.bazel b/third_party/zigimg/zigimg.bazel new file mode 100644 index 00000000..f02d9380 --- /dev/null +++ b/third_party/zigimg/zigimg.bazel @@ -0,0 +1,18 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "zigimg", + srcs = glob([ + "src/*.zig", + "src/compressions/*.zig", + "src/compressions/deflate/*.zig", + "src/formats/*.zig", + "src/formats/jpeg/*.zig", + "src/formats/png/*.zig", + "src/formats/tiff/*.zig", + "src/Image/*.zig", + "tests/**/*.zig", + ]), + main = "zigimg.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel b/third_party/zio/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel rename to third_party/zio/BUILD.bazel diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl new file mode 100644 index 00000000..96133ef0 --- /dev/null +++ b/third_party/zio/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "zio", + remote = "https://github.com/lalinsky/zio", + commit = "1b4e9787f8a4a67d91372a603b23258364f89bf0", + build_file = Label("//third_party/zio:zio.bazel"), + ) diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel new file mode 100644 index 00000000..39805936 --- /dev/null +++ b/third_party/zio/zio.bazel @@ -0,0 +1,46 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +string_flag( + name = "backend", + build_setting_default = "", + values = ["", "poll", "linux", "epoll", "kqueue", "io_uring", "iocp"], +) + +string_flag( + name = "scheduling", + build_setting_default = "work_stealing", + values = ["single_executor", "pinned", "work_stealing"], +) + +string_flag( + name = "resolve_beneath_mode", + build_setting_default = "strict", + values = ["strict", "best_effort"], +) + +zig_option(name = "backend_option", flag = ":backend", nullable = True) +zig_option(name = "scheduling_option", flag = ":scheduling") +zig_option(name = "resolve_beneath_mode_option", flag = ":resolve_beneath_mode") +zig_option(name = "no_hacks", bool = False) +zig_option(name = "scheduler_metrics", bool = False) + +zig_options( + name = "zio_build_options", + settings = [ + ":backend_option", + ":scheduling_option", + ":resolve_beneath_mode_option", + ":no_hacks", + ":scheduler_metrics", + ], +) + +zig_library( + name = "zio", + main = "src/zio.zig", + srcs = glob(["src/**/*.zig"]), + deps = [":zio_build_options"], + visibility = ["//visibility:public"], +) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index b47cae3e..69cc6128 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -1,51 +1,36 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary") -load(":resolved_toolchain.bzl", "resolved_toolchain") +load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig/zls:toolchain.bzl", "zls_toolchain") -# This is the ZLS binary toolchain type. Zig rule authors should use this -# target in the `toolchains` parameter in order to access the appropriate ZLS -# binary for the current execution platform. -toolchain_type( - name = "toolchain_type", - visibility = ["//visibility:public"], +exports_files(glob(["*.bazel"])) + +zig_library( + name = "build_options", + main = "build_options.zig", + visibility = ["@zls//:__pkg__"], ) -resolved_toolchain( - name = "resolved_toolchain", - # Marked manual so that `bazel test //...` passes - # even if no toolchain is registered. - tags = ["manual"], - visibility = ["//visibility:public"], +zig_library( + name = "exe_options", + main = "exe_options.zig", + visibility = ["@zls//:__pkg__"], ) -exports_files( - [ - "workspace_printer.zig", - "zls_build_runner.zig", - "zls_runner.zig", - "zls.completion.tpl", - ], +zig_library( + name = "tracy_options", + import_name = "options", + main = "tracy_options.zig", + visibility = ["@zls//:__pkg__"], ) -# Execute `bazel run //util:update_filegroups` to update this target. -filegroup( - name = "all_files", - srcs = [ - ":BUILD.bazel", - ":zls.completion.tpl", - ":zls_build_runner.zig", - ":zls_completion.bzl", - ":zls_runner.zig", - ":zls_toolchain.bzl", - ], - visibility = ["@rules_zig//zig:__pkg__"], +zls_toolchain( + name = "zls", + bin = "@zls//:zls", + zls_version = "0.17.0", ) -zig_binary( - name = "workspace_printer", - main = "workspace_printer.zig", -) - -zig_binary( - name = "build_runner", - main = "zls_build_runner.zig", +toolchain( + name = "toolchain", + target_settings = ["@zig_toolchains//:any_0.17.release"], + toolchain = ":zls", + toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) diff --git a/third_party/zls/build_options.zig b/third_party/zls/build_options.zig new file mode 100644 index 00000000..b50cb2d8 --- /dev/null +++ b/third_party/zls/build_options.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub const version_string = "0.17.0-dev+eab2be0fd744"; +pub const version = std.SemanticVersion.parse(version_string) catch unreachable; +pub const minimum_runtime_zig_version_string = "0.17.0"; diff --git a/third_party/zls/diffz.bazel b/third_party/zls/diffz.bazel new file mode 100644 index 00000000..fb4b85fa --- /dev/null +++ b/third_party/zls/diffz.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "diffz", + srcs = glob(["**/*.zig"]), + main = "DiffMatchPatch.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/exe_options.zig b/third_party/zls/exe_options.zig new file mode 100644 index 00000000..eaad61b5 --- /dev/null +++ b/third_party/zls/exe_options.zig @@ -0,0 +1,3 @@ +pub const enable_failing_allocator = false; +pub const enable_failing_allocator_likelihood: u32 = 256; +pub const debug_gpa = false; diff --git a/third_party/zls/known_folders.bazel b/third_party/zls/known_folders.bazel new file mode 100644 index 00000000..234db4b0 --- /dev/null +++ b/third_party/zls/known_folders.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "known-folders", + srcs = glob(["**/*.zig"]), + main = "known-folders.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/lsp.bazel b/third_party/zls/lsp.bazel new file mode 100644 index 00000000..b7c6b6dd --- /dev/null +++ b/third_party/zls/lsp.bazel @@ -0,0 +1,51 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# @embedFile resolves a module's root file without parsing it as Zig. +genrule( + name = "meta_model_source", + srcs = ["metaModel.json"], + outs = ["meta_model.zig"], + cmd = "cp $(<) $(@)", +) + +zig_library( + name = "meta-model", + main = ":meta_model.zig", +) + +zig_binary( + name = "codegen", + srcs = glob(["src/codegen/*.zig"]), + extra_srcs = ["src/codegen/config.zon"], + main = "src/codegen/codegen.zig", + deps = [":meta-model"], +) + +genrule( + name = "generate_types", + outs = ["lsp_types.zig"], + cmd = "$(location :codegen) $(@)", + tools = [":codegen"], +) + +zig_library( + name = "parser", + main = "src/parser.zig", +) + +zig_library( + name = "types", + main = ":lsp_types.zig", + deps = [":parser"], +) + +zig_library( + name = "lsp", + srcs = glob(["src/*.zig"]), + main = "src/lsp.zig", + visibility = ["@zls//:__pkg__"], + deps = [ + ":parser", + ":types", + ], +) diff --git a/third_party/zls/repo.bzl b/third_party/zls/repo.bzl new file mode 100644 index 00000000..27f05c5c --- /dev/null +++ b/third_party/zls/repo.bzl @@ -0,0 +1,28 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # ZLS 0.17 development revision; dependencies match its build.zig.zon. + git_repository( + name = "zls", + remote = "https://github.com/zigtools/zls.git", + commit = "eab2be0fd74443a27662da809b771c4ad0d2afcf", + build_file = Label("//third_party/zls:zls.bazel"), + ) + git_repository( + name = "zls_lsp", + remote = "https://github.com/zigtools/lsp-kit.git", + commit = "d148676ace1eaecf3c703ce7d350d0e94e502ed7", + build_file = Label("//third_party/zls:lsp.bazel"), + ) + git_repository( + name = "zls_diffz", + remote = "https://github.com/ziglibs/diffz.git", + commit = "aac8aa99c436ab8277b0711922aad062c0167b12", + build_file = Label("//third_party/zls:diffz.bazel"), + ) + git_repository( + name = "zls_known_folders", + remote = "https://github.com/ziglibs/known-folders.git", + commit = "6da0d0c41b78b9ed2d34fa364fcb81b5ebece6c4", + build_file = Label("//third_party/zls:known_folders.bazel"), + ) diff --git a/third_party/zls/resolved_toolchain.bzl b/third_party/zls/resolved_toolchain.bzl deleted file mode 100644 index c5e38e63..00000000 --- a/third_party/zls/resolved_toolchain.bzl +++ /dev/null @@ -1,24 +0,0 @@ -"""This module implements an alias rule to the resolved toolchain.""" - -DOC = """\ -Exposes a concrete toolchain which is the result of Bazel resolving the -toolchain for the execution or target platform. -Workaround for https://github.com/bazelbuild/bazel/issues/14009 -""" - -# Forward all the providers -def _resolved_toolchain_impl(ctx): - toolchain_info = ctx.toolchains["//third_party/zls:toolchain_type"] - return [ - toolchain_info, - toolchain_info.default, - toolchain_info.zlstoolchaininfo, - ] - -# Copied from java_toolchain_alias -# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl -resolved_toolchain = rule( - implementation = _resolved_toolchain_impl, - toolchains = ["//third_party/zls:toolchain_type"], - doc = DOC, -) diff --git a/third_party/zls/string_replace_reader.zig b/third_party/zls/string_replace_reader.zig deleted file mode 100644 index 3b68bf67..00000000 --- a/third_party/zls/string_replace_reader.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -pub const StringReplaceReader = struct { - interface: std.Io.Reader = .{ - .buffer = &.{}, - .vtable = &.{ - .stream = stream, - }, - .seek = 0, - .end = 0, - }, - in: *std.Io.Reader, - pattern: []const u8, - replace_by: []const u8, - - pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *StringReplaceReader = @fieldParentPtr("interface", r); - if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { - self.in.toss(self.pattern.len); - try w.writeAll(self.replace_by); - return self.replace_by.len; - } - return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { - std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, - else => |e| return e, - }; - } -}; diff --git a/third_party/zls/tracy_options.zig b/third_party/zls/tracy_options.zig new file mode 100644 index 00000000..01dfcb1d --- /dev/null +++ b/third_party/zls/tracy_options.zig @@ -0,0 +1,3 @@ +pub const enable = false; +pub const enable_allocation = false; +pub const enable_callstack = false; diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig deleted file mode 100644 index cbd8e130..00000000 --- a/third_party/zls/workspace_printer.zig +++ /dev/null @@ -1,39 +0,0 @@ -const std = @import("std"); - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const io = init.io; - const args = try init.minimal.args.toSlice(arena.allocator()); - - const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - var output_base_result = try std.process.run(arena.allocator(), init.io, .{ - .argv = &.{ - "bazel", - "info", - "execution_root", - }, - .cwd = .{ .path = build_workspace_directory }, - }); - - var output = try std.Io.Dir.cwd().readFileAlloc( - io, - args[1], - arena.allocator(), - .unlimited, - ); - output = try std.mem.replaceOwned( - u8, - arena.allocator(), - output, - "@@__BUILD_WORKSPACE_DIRECTORY__@@", - build_workspace_directory, - ); - output = try std.mem.replaceOwned( - u8, - arena.allocator(), - output, - "@@__BAZEL_EXECUTION_ROOT__@@", - std.mem.trimEnd(u8, output_base_result.stdout, "\n"), - ); - try std.Io.File.stdout().writeStreamingAll(io, output); -} diff --git a/third_party/zls/zls.bazel b/third_party/zls/zls.bazel new file mode 100644 index 00000000..56ccbff8 --- /dev/null +++ b/third_party/zls/zls.bazel @@ -0,0 +1,53 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +zig_binary( + name = "config_gen", + extra_srcs = ["src/tools/config.json"], + main = "src/tools/config_gen.zig", +) + +genrule( + name = "generate_version_data", + srcs = ["src/tools/langref.html.in"], + outs = ["version_data.zig"], + cmd = "$(location :config_gen) --langref-version master --langref-path $(<) --generate-version-data $(@)", + tools = [":config_gen"], +) + +zig_library( + name = "version_data", + main = ":version_data.zig", +) + +zig_library( + name = "tracy", + main = "src/tracy.zig", + deps = ["@zml//third_party/zls:tracy_options"], +) + +zig_library( + name = "zls_lib", + srcs = glob(["src/**/*.zig"]), + import_name = "zls", + main = "src/zls.zig", + deps = [ + ":tracy", + ":version_data", + "@zls_diffz//:diffz", + "@zls_lsp//:lsp", + "@zml//third_party/zls:build_options", + ], +) + +zig_binary( + name = "zls", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + visibility = ["//visibility:public"], + deps = [ + ":tracy", + ":zls_lib", + "@zls_known_folders//:known-folders", + "@zml//third_party/zls:exe_options", + ], +) diff --git a/third_party/zls/zls.completion.tpl b/third_party/zls/zls.completion.tpl deleted file mode 100644 index b3fb688e..00000000 --- a/third_party/zls/zls.completion.tpl +++ /dev/null @@ -1,10 +0,0 @@ -{ - "dependencies": {}, - "modules": [ - @@PACKAGES@@ - ], - "include_dirs": [], - "top_level_steps": [], - "available_options": {}, - "c_macros": [] -} diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig deleted file mode 100644 index ce82a66d..00000000 --- a/third_party/zls/zls_build_runner.zig +++ /dev/null @@ -1,14 +0,0 @@ -const std = @import("std"); - -pub fn main(init: std.process.Init) !void { - const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; - var child = try std.process.spawn(init.io, .{ - .argv = &.{ - "bazel", - "run", - "@@__TARGET__@@", - }, - .cwd = .{ .path = build_workspace_directory }, - }); - _ = try child.wait(init.io); -} diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl deleted file mode 100644 index 3b84dbad..00000000 --- a/third_party/zls/zls_completion.bzl +++ /dev/null @@ -1,76 +0,0 @@ -"""Implementation of the zls_completion macro.""" - -load("@aspect_bazel_lib//lib:utils.bzl", "utils") -load("@bazel_skylib//rules:expand_template.bzl", "expand_template") -load("@rules_zig//zig:defs.bzl", "zig_binary") -load(":zls_write_build_config.bzl", "zls_write_build_config") -load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") - -def zls_completion(name, deps, testonly = False, **kwargs): - """Entry point for ZLS completion. - - Args: - name: The name of the completion target. - deps: The List of Zig modules to include for completion. - **kwargs: Additional keyword arguments passed to the `zig_binary` - """ - - # Generate the ZLS BuildConfig file. - # It contains the list of Zig packages alongside their main Zig file paths. - build_config = name + ".build_config" - zls_write_build_config( - name = build_config, - out = name + ".build_config.json", - deps = deps, - testonly = testonly, - ) - - # Create a target that will be invoked by ZLS using our customer build_runner. - build_config_printer = "{}.print_build_config".format(name) - zig_binary( - name = build_config_printer, - main = Label("//third_party/zls:workspace_printer.zig"), - data = [ - ":{}".format(build_config), - ], - args = [ - "$(location :{})".format(build_config), - ], - visibility = ["//visibility:private"], - testonly = testonly, - ) - - # Generate the Zig build runner that will be used by ZLS to query the build config. - expand_template( - name = name + ".build_runner", - out = name + ".build_runner.zig", - substitutions = { - "@@__TARGET__@@": str(utils.to_label(build_config_printer)), - }, - template = Label(":zls_build_runner.zig"), - testonly = testonly, - ) - - # Generate the Zig source file for the ZLS runner binary which embeds the - # rlocationpath of all runtime dependencies of the ZLS runner binary. - zls_write_runner_zig_src( - name = name + ".runner", - out = name + ".runner.zig", - build_runner = ":" + name + ".build_runner.zig", - testonly = testonly, - ) - - zig_binary( - name = name, - main = name + ".runner", - data = [ - "@rules_zig//zig:resolved_toolchain", - "@zml//third_party/zls:resolved_toolchain", - name + ".build_runner", - ], - deps = [ - "@rules_zig//zig/runfiles", - ], - testonly = testonly, - **kwargs - ) diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig deleted file mode 100644 index c0808d9e..00000000 --- a/third_party/zls/zls_runner.zig +++ /dev/null @@ -1,109 +0,0 @@ -/// Custom ZLS launcher. -/// -/// Sets up paths to ZLS dependencies with Bazel runfiles. -/// -/// This file is used as a template by `zls_write_runner_zig_src.bzl`. -const std = @import("std"); - -const bazel_builtin = @import("bazel_builtin"); -const runfiles = @import("runfiles"); - -fn getRandomFilename(io: std.Io, buf: []u8, extension: []const u8) ![]const u8 { - const now = std.Io.Clock.real.now(io); - return std.fmt.bufPrint(buf, "/tmp/{d}{s}", .{ now.nanoseconds, extension }) catch @panic("OOM"); -} - -const Config = struct { - /// Override the Zig library path. Will be automatically resolved using the 'zig_exe_path'. - zig_lib_path: ?[]const u8 = null, - - /// Specify the path to the Zig executable (not the directory). If unset, zig is looked up in `PATH`. e.g. `/path/to/zig-templeos-armless-1.0.0/zig`. - zig_exe_path: ?[]const u8 = null, - - /// Specify a custom build runner to resolve build system information. - build_runner_path: ?[]const u8 = null, - - /// Path to a directory that will be used as zig's cache. Will default to `${KnownFolders.Cache}/zls`. - global_cache_path: ?[]const u8 = null, -}; - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const io = init.io; - - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io, .environ_map = init.environ_map, .argv = init.minimal.args }) orelse - return error.RunfilesNotFound; - const r = r_.withSourceRepo(bazel_builtin.current_repository); - - const zls_bin_rpath = "@@__ZLS_BIN_RPATH__@@"; - var zls_bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zls_bin_path = try r.rlocation(zls_bin_rpath, &zls_bin_path_buf) orelse - return error.RLocationNotFound; - - const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; - var zig_exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zig_exe_path = blk2: { - if (zig_exe_rpath[0] == '/') { - break :blk2 zig_exe_rpath; - } else { - break :blk2 try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse - return error.RLocationNotFound; - } - }; - - const zig_lib_path = "@@__ZIG_LIB_PATH__@@"; - const zig_lib_computed_path = blk: { - if (zig_lib_path[0] == '/') { - break :blk zig_lib_path; - } else { - break :blk null; - } - }; - - const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; - var zls_build_runner_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const zls_build_runner_path = try r.rlocation(zls_build_runner_rpath, &zls_build_runner_path_buf) orelse - return error.RLocationNotFound; - - const global_cache_path = "@@__GLOBAL_CACHE_PATH__@@"; - - const config: Config = .{ - .zig_exe_path = zig_exe_path, - .zig_lib_path = zig_lib_computed_path, - .build_runner_path = zls_build_runner_path, - .global_cache_path = global_cache_path, - }; - - var buf: [std.Io.Dir.max_name_bytes]u8 = undefined; - const tmp_file_path = try getRandomFilename(io, &buf, ".json"); - - { - var tmp_file = try std.Io.Dir.createFileAbsolute(io, tmp_file_path, .{}); - defer tmp_file.close(io); - - var out_buf: [4096]u8 = undefined; - var tmp_file_writer = tmp_file.writer(io, &out_buf); - - const formatter = std.json.fmt(config, .{ .whitespace = .indent_2 }); - - try tmp_file_writer.interface.print("{f}", .{formatter}); - try tmp_file_writer.interface.flush(); - } - - const args = init.minimal.args.toSlice(arena.allocator()) catch return error.ArgsAllocFailed; - - const exec_args_len = args.len - 1 + 3; // Skip args[0] and add "zls" + --config-path + tmp_file_path - var exec_args = try arena.allocator().alloc([]const u8, exec_args_len); - - exec_args[0] = zls_bin_path; // "zls" - - @memcpy(exec_args[1 .. exec_args.len - 2], args[1..]); - @memcpy(exec_args[exec_args.len - 2 ..], &[_][]const u8{ - "--config-path", - tmp_file_path, - }); - var child = try std.process.spawn(io, .{ - .argv = exec_args, - }); - _ = try child.wait(io); -} diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl deleted file mode 100644 index d83e8f1a..00000000 --- a/third_party/zls/zls_toolchain.bzl +++ /dev/null @@ -1,123 +0,0 @@ -"""Implementation of the zls_toolchain rule.""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_VERSION = "0.16.0-dev.215+0e289f5c" - -_ZLS_PLATFORMS = { - "x86_64-linux": struct( - compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - ), - "x86_64-macos": struct( - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:x86_64", - ], - ), - "aarch64-macos": struct( - compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:aarch64", - ], - ), -} - -_ZLS_VERSIONS = { - "x86_64-linux": "026a0ce9ea06b276319210509fa2a2edd7acf4e68da83b1a704c06ae26671548", - "x86_64-macos": "061845ec346620b914063763e8e39f9a374d2817c4ae976ea3eeb577a6e1cf32", - "aarch64-macos": "fb711e068e2ec77b3979458ff72f18311fa7df74644a8cf43985c0f88cae5c66", -} - -ZlsToolchainInfo = provider( - doc = "Provide info for executing zls", - fields = { - "bin": "Executable ZLS binary", - }, -) - -def _zls_toolchain_impl(ctx): - binary = ctx.file.bin - - default_info = DefaultInfo( - files = depset([binary]), - runfiles = ctx.runfiles(files = [binary]), - ) - zlstoolchaininfo = ZlsToolchainInfo( - bin = binary, - ) - toolchain_info = platform_common.ToolchainInfo( - default = default_info, - zlstoolchaininfo = zlstoolchaininfo, - ) - - return [ - default_info, - zlstoolchaininfo, - toolchain_info, - ] - -zls_toolchain = rule( - implementation = _zls_toolchain_impl, - attrs = { - "bin": attr.label( - executable = True, - allow_single_file = True, - cfg = "exec", - ), - }, -) - -def _zls_toolchains_repos_impl(mctx): - for platform, _ in _ZLS_PLATFORMS.items(): - http_archive( - name = "zls_{}".format(platform), - url = "https://builds.zigtools.org/zls-{platform}-{version}.tar.xz".format( - version = _VERSION, - platform = platform, - ), - sha256 = _ZLS_VERSIONS[platform], - build_file_content = """\ -load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") -zls_toolchain(name = "toolchain", bin = "zls") -""", - ) - - zls_toolchains_repo(name = "zls_toolchains") - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["zls_toolchains"], - root_module_direct_dev_deps = [], - ) - -zls_toolchains = module_extension( - implementation = _zls_toolchains_repos_impl, -) - -def _zls_toolchains_repo_impl(rctx): - build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" - - for platform, config in _ZLS_PLATFORMS.items(): - build_content += """ -toolchain( - name = "{platform}_toolchain", - exec_compatible_with = {compatible_with}, - target_compatible_with = {compatible_with}, - toolchain = "@zls_{platform}//:toolchain", - toolchain_type = "@zml//third_party/zls:toolchain_type", -) -""".format( - platform = platform, - compatible_with = config.compatible_with, - ) - - rctx.file("BUILD.bazel", build_content) - -zls_toolchains_repo = repository_rule( - _zls_toolchains_repo_impl, - doc = """Creates a repository with toolchain definitions for all known platforms - which can be registered or selected.""", -) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl deleted file mode 100644 index d7260de2..00000000 --- a/third_party/zls/zls_write_build_config.bzl +++ /dev/null @@ -1,176 +0,0 @@ -"""Generates the ZLS build config file.""" - -load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") -load("@rules_cc//cc/common:cc_common.bzl", "cc_common") -load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") -load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") -load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") -load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") -load("@rules_zig//zig/private/common:zig_lib_dir.bzl", "zig_lib_dir") -load("@rules_zig//zig/private/providers:zig_module_info.bzl", "ZigModuleInfo", "zig_module_info") -load( - "@rules_zig//zig/private/providers:zig_target_info.bzl", - "zig_target_platform", -) - -def _zls_construct_zig_module_info_impl(target, ctx): - """Aspect that constructs ZigModuleInfo for zig_binary and zig_library rules.""" - if ZigModuleInfo in target: - return [] - - if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library", "zig_test"): - return [] - - cdeps = [] - zdeps = [] - for dep in ctx.rule.attr.deps: - if ZigModuleInfo in dep: - zdeps.append(dep[ZigModuleInfo]) - elif CcInfo in dep: - cdeps.append(dep[CcInfo]) - - root_module_is_only_dep = len(ctx.rule.attr.deps) == 1 and ZigModuleInfo in ctx.rule.attr.deps[0] - if root_module_is_only_dep: - root_module = ctx.rule.attr.deps[0][ZigModuleInfo] - else: - root_module = zig_module_info( - name = ctx.rule.attr.name, - canonical_name = target.label.name, - main = ctx.rule.file.main, - srcs = ctx.rule.files.srcs, - extra_srcs = ctx.rule.files.extra_srcs, - deps = zdeps, # [bazel_builtin_module(ctx)], - cdeps = cdeps, - zigopts = [], - ) - - return [ - root_module, - ] - -zls_construct_zig_module_info = aspect( - implementation = _zls_construct_zig_module_info_impl, -) - -def format_main_file(main): - prefix = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" - if (main.startswith("bazel-out/") or main.startswith("external/")): - prefix = "@@__BAZEL_EXECUTION_ROOT__@@/" - return prefix + main - -def _zls_write_build_config_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zigtargetinfo = ctx.toolchains["@rules_zig//zig/target:toolchain_type"].zigtargetinfo - - c_module_contexts = [] - c_module_inputs = [] - cc_info = cc_common.merge_cc_infos(direct_cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) - if need_translate_c(cc_info): - global_args = ctx.actions.args() - global_args.use_param_file("@%s") - - zig_target_platform( - target = zigtargetinfo, - args = global_args, - ) - - zig_lib_dir( - zigtoolchaininfo = zigtoolchaininfo, - args = global_args, - ) - - zig_cache_output( - zigtoolchaininfo = zigtoolchaininfo, - args = global_args, - ) - - c_module = zig_translate_c( - ctx = ctx, - name = "c", - canonical_name = "c", - zigtoolchaininfo = zigtoolchaininfo, - global_args = global_args, - cc_infos = [cc_info], - ) - - c_module_contexts = [depset( - direct = [c_module.module_context], - transitive = [c_module.transitive_module_contexts], - )] - c_module_inputs = [c_module.transitive_inputs] - - contexts = depset( - direct = [dep[ZigModuleInfo].module_context for dep in ctx.attr.deps], - transitive = [dep[ZigModuleInfo].transitive_module_contexts for dep in ctx.attr.deps] + c_module_contexts, - ) - - modules = { - mod.canonical_name: mod - for mod in contexts.to_list() - } - - output = { - "dependencies": {}, - "modules": { - format_main_file(mod.main): { - "import_table": { - dep.name: format_main_file(modules.get(dep.canonical_name, "").main) - for dep in mod.dependency_mappings - }, - "c_macros": [], - "include_dirs": [], - } - for mod in modules.values() - }, - "compilations": [], - "top_level_steps": [], - "available_options": {}, - } - - config = ctx.outputs.out - ctx.actions.write( - output = config, - content = json.encode(output), - ) - - return [ - DefaultInfo( - files = depset(direct = [config]), - # Not really runfiles but we need to pull the output of translate_c... - runfiles = ctx.runfiles(transitive_files = depset(transitive = c_module_inputs)), - ), - ] - -zls_write_build_config = rule( - implementation = _zls_write_build_config_impl, - attrs = { - "deps": attr.label_list( - providers = [ZigModuleInfo], - mandatory = True, - aspects = [zls_construct_zig_module_info], - ), - "out": attr.output( - mandatory = True, - ), - "_zls_completion_tpl": attr.label( - default = ":zls.completion.tpl", - allow_single_file = True, - ), - "_translate_c": attr.label( - default = Label("@rules_zig//zig/private/common:translate-c"), - cfg = "exec", - executable = True, - ), - "_c_helpers": attr.label( - default = Label("@rules_zig//zig/private/common:helpers"), - ), - "_c_builtins": attr.label( - default = Label("@rules_zig//zig/private/common:c_builtins"), - ), - }, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "@rules_zig//zig/target:toolchain_type", - ] + use_cc_toolchain(mandatory = False), - fragments = ["cpp", "apple"], -) diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl deleted file mode 100644 index ac89e08e..00000000 --- a/third_party/zls/zls_write_runner_zig_src.bzl +++ /dev/null @@ -1,45 +0,0 @@ -"""Generates the ZLS runner source file.""" - -load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path") - -def _zls_write_runner_zig_src_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zlstoolchaininfo = ctx.toolchains["//third_party/zls:toolchain_type"].zlstoolchaininfo - - zls_runner = ctx.outputs.out - ctx.actions.expand_template( - output = zls_runner, - template = ctx.file._runner_tpl, - substitutions = { - "@@__ZIG_EXE_RPATH__@@": zigtoolchaininfo.zig_exe_rpath, - "@@__ZIG_LIB_PATH__@@": zigtoolchaininfo.zig_lib_rpath, - "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), - "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), - "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, - }, - ) - - return [ - DefaultInfo(files = depset([zls_runner])), - ] - -zls_write_runner_zig_src = rule( - implementation = _zls_write_runner_zig_src_impl, - attrs = { - "out": attr.output( - mandatory = True, - ), - "build_runner": attr.label( - allow_single_file = True, - mandatory = True, - ), - "_runner_tpl": attr.label( - default = "zls_runner.zig", - allow_single_file = True, - ), - }, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "//third_party/zls:toolchain_type", - ], -) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 5dc3d219..844778f5 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,7 +1,26 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") py_library( name = "zml_utils", srcs = ["zml_utils.py"], visibility = ["//visibility:public"], ) + +sh_binary( + name = "nsys", + srcs = ["nsys.sh"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "rocprofv3", + srcs = ["rocprofv3.sh"], + data = [ + "@amdrocm-profiler-base//:rocprofv3_tool", + "@libzml_rocm//:sandbox", + ], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], +) diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel new file mode 100644 index 00000000..8bfcabed --- /dev/null +++ b/tools/neuron/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + +sh_binary( + name = "profile", + srcs = ["profile.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "server", + srcs = ["server.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "3001", + ], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "ingest", + srcs = ["ingest.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "/tmp/zml-neuron-profile", + ], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-txt", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-json", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-perfetto", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) diff --git a/tools/neuron/ingest.sh b/tools/neuron/ingest.sh new file mode 100755 index 00000000..bb03850d --- /dev/null +++ b/tools/neuron/ingest.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" +data_path="$1" +default_profile_root="$2" +shift 2 + +mkdir -p "${data_path}" + +ingest_run() { + local run_dir="$1" + local execution_dir="${run_dir}/execution" + local run_id + run_id="$(basename "${run_dir}")" + + if [[ ! -d "${execution_dir}" ]]; then + echo "skipping ${run_dir}: missing execution directory" >&2 + return + fi + + for profile_session in "${execution_dir}"/*/*; do + [[ -f "${profile_session}/ntrace.pb" ]] || continue + + local system_dir + system_dir="$(mktemp -d)" + for file in ntrace.pb trace_info.pb cpu_util.pb host_mem.pb; do + [[ -f "${profile_session}/${file}" ]] && cp "${profile_session}/${file}" "${system_dir}/${file}" + done + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${run_id}:system" \ + -d "${system_dir}" + + rm -rf "${system_dir}" + done + + for neff in "${execution_dir}"/*/*/*.neff; do + local neff_name + local program_name + local ntff_prefix + neff_name="$(basename "${neff}" .neff)" + program_name="$(grep -aEom1 'MODULE_[A-Za-z0-9_]+' "${neff}" | sed -E 's/^MODULE_//; s/_[0-9]+$//')" + if [[ -z "${program_name}" ]]; then + echo "failed to find program name in ${neff}" >&2 + exit 1 + fi + ntff_prefix="$(dirname "${neff}")/${neff_name}" + + if ! compgen -G "${ntff_prefix}_rank_*.ntff" >/dev/null && [[ ! -f "${ntff_prefix}.ntff" ]]; then + "${neuron_explorer_bin}" capture \ + -n "${neff}" \ + -s "${ntff_prefix}.ntff" \ + --num-exec=2 \ + --profile-nth-exec=2 \ + --io-from=neff + fi + + for ntff in "${ntff_prefix}"_rank_*.ntff "${ntff_prefix}".ntff; do + [[ -f "${ntff}" ]] || continue + + local core_suffix + local display_name + core_suffix="$(basename "${ntff}" .ntff | sed -E 's/^.*_rank_([0-9]+).*$/nc\1/; s/^.*_(vnc_[0-9]+)$/\1/')" + display_name="${run_id}:${program_name}_${core_suffix}" + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${display_name}" \ + -n "${neff}" \ + -s "${ntff}" + done + done +} + +[[ $# -eq 0 ]] && set -- "${default_profile_root}" + +for path in "$@"; do + if [[ -d "${path}/execution" ]]; then + ingest_run "${path}" + else + for run_dir in "${path}"/*; do + [[ -d "${run_dir}" ]] && ingest_run "${run_dir}" + done + fi +done diff --git a/tools/neuron/profile.sh b/tools/neuron/profile.sh new file mode 100755 index 00000000..e1118d56 --- /dev/null +++ b/tools/neuron/profile.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +profile_root="$1" +shift + +run_dir="${profile_root}/$(date -u '+%Y%m%dT%H%M%SZ')" +execution_dir="${run_dir}/execution" +compile_dir="${run_dir}/compile" + +mkdir -p "${execution_dir}" "${compile_dir}" + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +xla_flags="--xla_dump_to=${compile_dir} --xla_dump_hlo_as_text --xla_dump_hlo_as_proto --xla_dump_hlo_pass_re=.*" + +set +e +env \ + NEURON_RT_INSPECT_ENABLE=1 \ + NEURON_RT_INSPECT_SYSTEM_PROFILE=1 \ + NEURON_RT_INSPECT_DEVICE_PROFILE=1 \ + NEURON_RT_INSPECT_OUTPUT_DIR="${execution_dir}" \ + SKIP_PJRT_PROFILER=true \ + XLA_IR_DEBUG=1 \ + XLA_HLO_DEBUG=1 \ + XLA_FLAGS="${xla_flags}" \ + "${neuron_explorer_bin}" inspect -o "${execution_dir}" "$@" +status=$? +set -e + +echo "Neuron profile dump path: ${run_dir}" >&2 + +exit "${status}" diff --git a/tools/neuron/server.sh b/tools/neuron/server.sh new file mode 100755 index 00000000..3d0babe1 --- /dev/null +++ b/tools/neuron/server.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +data_path="$1" +port="$2" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +mkdir -p "${data_path}" + +echo "Neuron Explorer server: http://127.0.0.1:${port}" + +exec "${neuron_explorer_bin}" view \ + --data-path "${data_path}" \ + --port "${port}" diff --git a/tools/neuron/summary.sh b/tools/neuron/summary.sh new file mode 100755 index 00000000..5a1b1461 --- /dev/null +++ b/tools/neuron/summary.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +run_dir="$1" +execution_dir="${run_dir}/execution" + +case "$(basename "$0")" in + summary-txt) + output_format="summary-text" + output_file="summary.txt" + ;; + summary-json) + output_format="summary-json" + output_file="summary.json" + ;; + summary-perfetto) + output_format="perfetto" + output_file="summary.pftrace" + ;; +esac + +if [[ "${output_format}" == "summary-text" || "${output_format}" == "summary-json" ]]; then + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + >"${run_dir}/${output_file}" +else + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + --output-file "${run_dir}/${output_file}" +fi + +echo "Neuron summary path: ${run_dir}/${output_file}" >&2 diff --git a/tools/nsys.sh b/tools/nsys.sh new file mode 100755 index 00000000..b8d38e38 --- /dev/null +++ b/tools/nsys.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +sudo_cmd=() +if [[ "${ZML_PROFILE_NO_SUDO:-0}" != "1" ]]; then + sudo_cmd=(sudo -E) +fi + +declare -a nsys_args +if [[ -n "${ZML_NSYS_ARGS:-}" ]]; then + nsys_args=(${ZML_NSYS_ARGS}) +else + nsys_args=( + profile + -t + cuda,syscall,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn,osrt + --inherit-environment=true + --cuda-memory-usage + true + --cuda-event-trace=false + --backtrace=dwarf + --cuda-graph-trace=node + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${sudo_cmd[@]}" nsys "${nsys_args[@]}" "$@" diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh new file mode 100755 index 00000000..cc7e70bb --- /dev/null +++ b/tools/rocprofv3.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +if [[ -n "${RUNFILES_DIR:-}" ]]; then + runfiles_root="${RUNFILES_DIR}" +elif [[ -d "$0.runfiles" ]]; then + runfiles_root="$0.runfiles" +else + echo "failed to locate Bazel runfiles for sandboxed rocprofv3" >&2 + exit 1 +fi + +rocm_root="${runfiles_root}/+rocm_packages+libzml_rocm/sandbox" +if [[ ! -d "${rocm_root}" ]]; then + echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 + exit 1 +fi + +rocprofv3_bin="${rocm_root}/bin/rocprofv3" +if [[ ! -x "${rocprofv3_bin}" ]]; then + echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 + exit 1 +fi + +declare -a rocprof_args +if [[ -n "${ZML_ROCPROFV3_ARGS:-}" ]]; then + rocprof_args=(${ZML_ROCPROFV3_ARGS}) +else + rocprof_args=( + --sys-trace + --runtime-trace + --hip-trace + --kernel-trace + --memory-copy-trace + --hip-runtime-trace + --hsa-core-trace + --scratch-memory-trace + --marker-trace + --kernel-rename + --stats + --output-format + pftrace + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${rocprofv3_bin}" --rocm-root "${rocm_root}" "${rocprof_args[@]}" -- "$@" diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel new file mode 100644 index 00000000..aff4954d --- /dev/null +++ b/tools/safetensors/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "safetensors", + main = "main.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + "//stdx", + "//vfs", + "//zml", + ], +) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig new file mode 100644 index 00000000..cf902eef --- /dev/null +++ b/tools/safetensors/main.zig @@ -0,0 +1,91 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zml = @import("zml"); + +const log = std.log.scoped(.safetensors_convert); + +const Args = struct { + output: ?[]const u8 = null, + positional: struct { model: []const u8 }, + + pub const help = + \\Usage: [--output=] + \\ + \\Read safetensors headers and write model.safetensors.full_index.json in the working directory + \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash + ; +}; + +pub fn main(init: std.process.Init) !void { + const args = stdx.flags.parse(init.minimal.args, Args); + const allocator = init.gpa; + + const start = std.Io.Clock.now(.awake, init.io); + var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer client.deinit(); + try client.initDefaultProxies(allocator, init.environ_map); + + const input = args.positional.model; + + var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); + defer file_vfs.deinit(); + var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); + defer http_vfs.deinit(); + var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); + defer https_vfs.deinit(); + var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (hf_vfs) |*backend| backend.deinit(); + var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (s3_vfs) |*backend| backend.deinit(); + var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (gcs_vfs) |*backend| backend.deinit(); + + var vfs: VFS = try .init(allocator, init.io); + const io = vfs.io(); + defer vfs.deinit(); + try vfs.register("file", file_vfs.io()); + try vfs.register("http", http_vfs.io()); + try vfs.register("https", https_vfs.io()); + if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); + if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); + if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const repo = try zml.safetensors.resolveModelRepo(io, input); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const loaded = std.Io.Clock.now(.awake, init.io); + + const index_json_file = if (args.output) |output| + try std.Io.Dir.cwd().createFile(io, output, .{}) + else + try repo.createFile(io, full_index_path, .{}); + const out_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(out_buffer); + var out_writer = index_json_file.writer(io, out_buffer); + try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); + const done = std.Io.Clock.now(.awake, init.io); + + log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ + args.output orelse full_index_path, + registry.tensors.count(), + start.durationTo(loaded), + loaded.durationTo(done), + start.durationTo(done), + }); +} + +const full_index_path = "model.safetensors.full_index.json"; diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh new file mode 100755 index 00000000..05f02bed --- /dev/null +++ b/tools/workspace_status.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if git diff --quiet; then + echo BUILD_SCM_REVISION "$(git rev-parse --short HEAD)" +else + echo BUILD_SCM_REVISION "dev" +fi + +echo STABLE_ZML_SMI_VERSION "$(git describe --tags --dirty --match 'zml-smi-v*' 2>/dev/null || echo "dev")" diff --git a/tools/xprof/BUILD.bazel b/tools/xprof/BUILD.bazel new file mode 100644 index 00000000..285f7003 --- /dev/null +++ b/tools/xprof/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary( + name = "xprof", + srcs = ["main.py"], + main = "main.py", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = ["@xprof//xprof:pkg"], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = ["--upgrade"], + tags = ["manual"], +) diff --git a/tools/xprof/main.py b/tools/xprof/main.py new file mode 100644 index 00000000..79ed3ccf --- /dev/null +++ b/tools/xprof/main.py @@ -0,0 +1,42 @@ +import os +import sys +from pathlib import Path + +from xprof import profile_plugin +from xprof.cli.xprof_cli import main as xprof_main + + +def _configure_static_dir() -> None: + static_dir = Path(profile_plugin.__file__).parent / "static" + # Bazel runfiles contain a real directory whose files are symlinks into the + # repository cache. Resolve a file so XProf's traversal check sees the same + # base path as the assets it opens. + resolved_static_dir = (static_dir / "index.html").resolve().parent + os.environ.setdefault("XPROF_STATIC_DIR", str(resolved_static_dir)) + + +def _argv() -> list[str]: + user_args = sys.argv[1:] + if user_args and not user_args[0].startswith("-"): + logdir = user_args[0] + trailing_args = user_args[1:] + else: + logdir = "." + trailing_args = user_args + return [ + sys.argv[0], + "--logdir", + logdir, + "--max_concurrent_worker_requests", + "32", + "--port", + "6006", + "--hide_capture_profile_button", + *trailing_args, + ] + + +if __name__ == "__main__": + _configure_static_dir() + sys.argv = _argv() + raise SystemExit(xprof_main()) diff --git a/tools/xprof/requirements.in b/tools/xprof/requirements.in new file mode 100644 index 00000000..7a238fea --- /dev/null +++ b/tools/xprof/requirements.in @@ -0,0 +1 @@ +xprof==2.23.2 diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt new file mode 100644 index 00000000..f3ac6542 --- /dev/null +++ b/tools/xprof/requirements.lock.txt @@ -0,0 +1,1453 @@ +# This file was autogenerated by uv via the following command: +# bazel run //tools/xprof:requirements.update +absl-py==2.5.0 \ + --hash=sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f \ + --hash=sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba + # via xprof +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + # via aiohttp +aiohttp==3.14.3 \ + --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \ + --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \ + --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \ + --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \ + --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \ + --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \ + --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \ + --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \ + --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \ + --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \ + --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \ + --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \ + --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \ + --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \ + --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \ + --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \ + --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \ + --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \ + --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \ + --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \ + --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \ + --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \ + --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \ + --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \ + --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \ + --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \ + --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \ + --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \ + --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \ + --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \ + --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \ + --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \ + --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \ + --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \ + --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \ + --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \ + --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \ + --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \ + --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \ + --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \ + --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \ + --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \ + --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \ + --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \ + --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \ + --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \ + --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \ + --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \ + --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \ + --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \ + --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \ + --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \ + --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \ + --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \ + --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \ + --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \ + --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \ + --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \ + --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \ + --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \ + --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \ + --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \ + --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \ + --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \ + --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \ + --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \ + --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \ + --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \ + --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \ + --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \ + --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \ + --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \ + --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \ + --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \ + --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \ + --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \ + --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \ + --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \ + --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \ + --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \ + --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \ + --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \ + --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \ + --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \ + --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \ + --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \ + --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \ + --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \ + --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \ + --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \ + --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \ + --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \ + --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \ + --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \ + --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \ + --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \ + --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \ + --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \ + --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \ + --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \ + --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \ + --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \ + --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \ + --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \ + --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \ + --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \ + --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \ + --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \ + --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \ + --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \ + --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \ + --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \ + --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \ + --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \ + --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \ + --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \ + --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \ + --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \ + --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5 + # via gcsfs +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via aiohttp +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f + # via requests +cheroot==11.1.2 \ + --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ + --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca + # via xprof +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via google-auth +decorator==5.3.1 \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c + # via gcsfs +etils[epath]==1.14.0 \ + --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ + --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 + # via xprof +fire==0.7.1 \ + --hash=sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf \ + --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 + # via xprof +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +fsspec[gcs]==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # etils + # gcsfs + # xprof +gcsfs==2026.8.1 \ + --hash=sha256:66e6316690b17e1887a4f4bc0902eee53f5b3fd860cc3f0377d65c5be3313b1f \ + --hash=sha256:738da7dcc74817c7668d032f932598482a23cce1b061c537e54278be44d2fff8 + # via fsspec +google-api-core[grpc]==2.37.0 \ + --hash=sha256:cf58f220aa797f1ffdda52194c4c7d72efeced09297c2976529f8135f6d85b9a \ + --hash=sha256:d84042a0034cce9c4304e17d2e5e9966c2fcb1a851b73799291f687a67470c8b + # via + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth==2.58.0 \ + --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ + --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth-oauthlib==1.4.1 \ + --hash=sha256:1a83f5f2a8421dedadaa3caf25b3a710dddf85a33a63144be41c2fc79174b106 \ + --hash=sha256:a1be43ec69fe563ac9b2c4d6fc4334b323b21cbdc59a638b5fa34dd4d5a2a348 + # via gcsfs +google-cloud-core==2.7.0 \ + --hash=sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d \ + --hash=sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1 + # via google-cloud-storage +google-cloud-storage==3.14.1 \ + --hash=sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118 \ + --hash=sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7 + # via + # gcsfs + # xprof +google-cloud-storage-control==1.15.0 \ + --hash=sha256:46c8c801ff0425a1bd59bc28899a8242e15f6bbb06a4f2304cbd169c7d26cb29 \ + --hash=sha256:e3d50276f0aed0c33ebcf3c6221e933889e042df70a56886ddffe59c66889183 + # via gcsfs +google-crc32c==1.8.0 \ + --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ + --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ + --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ + --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ + --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ + --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ + --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ + --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ + --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ + --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ + --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ + --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ + --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ + --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ + --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ + --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ + --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ + --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ + --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ + --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ + --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ + --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ + --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ + --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ + --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ + --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ + --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ + --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ + --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ + --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ + --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ + --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ + --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.10.2 \ + --hash=sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c \ + --hash=sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7 + # via google-cloud-storage +googleapis-common-protos[grpc]==1.75.3 \ + --hash=sha256:57c435ac2c68b108999b6db075d9053e4d7a936ba57b4a3d45667b1346f1738a \ + --hash=sha256:a018d2bf098ca9fb6faa08d5bb780e2a2c2f73c566f069761331386c9596d3f2 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +grpc-google-iam-v1==0.14.5 \ + --hash=sha256:07fd3a9fafb586588e771831fbfc8f6597050181d0c3b45e039d18b8fdc1aab5 \ + --hash=sha256:0f5e680b20aa0a9441e68c769da04d94d70fca4e43751a82d8abb8aa6a7181ca + # via google-cloud-storage-control +grpcio==1.84.0 \ + --hash=sha256:026d757df86c5b7a41de8200b9a2cda454aaa5004cb0c7e3374c66eb82f61499 \ + --hash=sha256:06619ba1515e5ee69fb2a514e95dd8be05ce74cb3928d5b34f87f87c86fe3c27 \ + --hash=sha256:08735e3d08d24ab3132cf87e2e5dea8746cabcc7d676c2b0b7362f195feef9d9 \ + --hash=sha256:0d532ade4486dad9b302ffa4d4683d67561051c26d17c4023322845e9fa10140 \ + --hash=sha256:158c1c11cfb61b4849c3caf4d52de6f5ecd376e14446feb4a90dc95a90d616f5 \ + --hash=sha256:15bb76489e337fc492685c9758e2fd4d4ab516b901ad830dc5a91987decf00be \ + --hash=sha256:19aaf172fc2edbefccce3f6e92c5150975dbe56c45744e9e87cf72ebdf85bfbe \ + --hash=sha256:209414080da8c20af94df1395b635da52dd57b5edc9e917e1deca0dc1c4bb55e \ + --hash=sha256:210e4c32f907045eb8158273e60c6ab69a3947697df6245dbda381f26c59485b \ + --hash=sha256:23e6e8e8a75cff88e0a793bfd3becea03a13e2763ae90c1ff573bc19ca5b429a \ + --hash=sha256:27b8b36200a9fbee6e120246f4a8a41657549107ef19fb2c819c4b2fd524f39a \ + --hash=sha256:28d2609691da93051e998495108bbddd2a9f7a561253bae94828d81290f30c15 \ + --hash=sha256:2c024da73b296f040b8360e60bd73a659b230093684a438da0e1260f34cc724e \ + --hash=sha256:393d8a78bff6731ecc5ad2151a821f8fbc1709b137ebb9c25a4ef399fbdcc914 \ + --hash=sha256:3d6a82c4fc6c85f2fb7572c86bdb86f84c97b6580e5f6599f711800bac48a5d8 \ + --hash=sha256:3de427b05f244ba2c2a9bdc67e7a6731c8340811524ecc4435466549f8af1d17 \ + --hash=sha256:406583b4e8fb2282ebd392e12b963e601c1f82e07125a8c2cb5b144e7e024796 \ + --hash=sha256:4119efa6519871719ad81f33bc95ab87857dcb1c5801f30a6e592f2c41164169 \ + --hash=sha256:42959bd50dd660ffc3f2a9bec15a6da4f9aaa0dda555d59ff2d2e80b908456a8 \ + --hash=sha256:455ed6083353b8e938f1d58c765eab2fbb165731e5b507be30fee344915a2a11 \ + --hash=sha256:465eef3d17e59ad22a556fc0138f7c7c799df426734344daec42c797d49fda99 \ + --hash=sha256:47ecf0d9b81d981f07b61bd89eced9d2582f5eaacc3aaa36ad27f81aef70a27f \ + --hash=sha256:49717e857899f4136d7657bf5aded61ac479110a075438290923a4d86af7cd02 \ + --hash=sha256:4aaeceeb7fa7d824c322d1ec3208c8495c88478a927295553235435fc49043ad \ + --hash=sha256:57dc36a5ab0e676f5f6e171de2917fd0aef73f32a9aaf23956bfe19997a30bd1 \ + --hash=sha256:5933a052946873d01a42119a05420d669bdca436aeba2d1851988ccb12b421c0 \ + --hash=sha256:5deda5b4bf62769eb98c119cca43d40e1231e34846b19db5cdea821d446a2253 \ + --hash=sha256:61386101ecaa096b694d0dd278caf99a56aeec78440cc17e918eef0b50f2d567 \ + --hash=sha256:659728f20fc7a0933ed7b1945435e31014b97ab8a5a7edcbaa70da4794aeb191 \ + --hash=sha256:70bb4ce8be0c5606bec259cbd7152374470396413b7863a658a08c849e6b29ff \ + --hash=sha256:71fd60e6e426d293d0a2f685115ad0a0845117602cf13605a4be7524fb5f7bba \ + --hash=sha256:756ea5c2da00fa65c930284892d2a9706828704ca3ba40b4c51c4834eb39fcfd \ + --hash=sha256:800b7e00d92553313c0463c200087930aa78678ec1d528193aeb50906f55989b \ + --hash=sha256:82da34ae4f639c73ac46e521e00c0a49bf86f717b9fb1f405f133e98731e38dc \ + --hash=sha256:8e1a45d174b6b8589f51dce1cea804aa6c1f72c9c80cba91ae2caabeb6d90540 \ + --hash=sha256:8e3f508d0e9e6236ba2f08d56e33355e434e785e813149a1b8477d3edf69779d \ + --hash=sha256:986e9751d416d7a6eaa2fecdac38da63153d63a4b340ba7d624889c490451500 \ + --hash=sha256:9b73836ba0e16fcbb57c31cf6cbc2907c8d8c790b83679df454b74bd15e0be04 \ + --hash=sha256:9bab4cf571653a8afffb83ce21aa27b51dfe629b526b7b6adec35491fe1fc2ea \ + --hash=sha256:a71d24f40b0cc6798feaa978c7411dc1135b7018e9fc0442db611c139bf58344 \ + --hash=sha256:a9383401d9f116f98cacd4eba6c505a6edb80ba65badfc8e8ed8ae64983bcc44 \ + --hash=sha256:b44f0a0fc7bc6677d38cc80bca1a32814ce6c8f200fb8b3c1a61c9d77eaefbf3 \ + --hash=sha256:b5c6f20d657ae09ae4e30d9d3a21edd13f1219d58cc6f999b9d1bb63be9c1baa \ + --hash=sha256:b61692f0069b3eee2fc8a3a1b7f6c044df9e03fede6ce69b3ca832e1c39f26c5 \ + --hash=sha256:b8c62888c3e49debf37ad9773e3c02f77b0c1e811f8fb0962f2b6c3bbab5b97a \ + --hash=sha256:bd8ea8eb3817b226057cc1c0e7ec4b378dcda52043b972b6ff12b1152178967d \ + --hash=sha256:c5559b492007dc09b4de9b95dab05f0b5e53547aad230cf07e46c7dd017a3be5 \ + --hash=sha256:d0fdd25faece8a1f95e8a3a8006e29701b5cf8dadb4a8132e68f3134637004a5 \ + --hash=sha256:e094dd21f077af8194923fc263cad872eaa1802bb0156fd7e5ae18e99cd86715 \ + --hash=sha256:e41c3993eee896c617dbd8a505085d28b6e84a0445ed9a1f40f95808473cf678 \ + --hash=sha256:e88d304f094f4937bc27ec6a435e218a084168f11ec630c8d5d39b431d08d81d \ + --hash=sha256:e90e3bdf7b5eac005fef631adae9cafde16f922def207b80a7c46b253c18ad20 \ + --hash=sha256:ed2c1493c44d0932f1e55fdb5d1ead658c68288ec5d51b8c4928422d98633ef9 \ + --hash=sha256:edb6f87fc60ff438557291501b3e16c7a77c3b01a52d782cf276dccc7c5dd89c \ + --hash=sha256:efb29f8633bf6630dc89de4fe0353ac3d7e4b70ef7b6e29fb40f00e68c127fa5 \ + --hash=sha256:f6c972474ce691aca74e58d17625450cef153dc4760364cadeb167983ea6d589 \ + --hash=sha256:f6d178ba6dc8e82976c184b65fddde172d054c17237993a3e083efe4f134d55b \ + --hash=sha256:f9a456bdbed52a01c9ab8423bdebab04a5363c78676edc55ab9b58bd13bdf9e1 \ + --hash=sha256:fbdbcd06986ede3ce584083b1dc2afe6808e8943e5cf50ad11183c03aceda25a \ + --hash=sha256:fc66cb50c93554b86db0b6625ab5c6e9051dbf8847c08d93c84918e02e413fb7 \ + --hash=sha256:fff5ef3fe1bba7d6147e5f19e01e5e122ac2c076486887ddcb8d42e663400fbe + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status +grpcio-status==1.84.0 \ + --hash=sha256:0c182ca0d6e60acbfd0e14499cf39a155e4827a1c3fd9f7638e49af15a74c30a \ + --hash=sha256:5caf28ba7184b81f618b5f7f094859fd2541bf429d2189bbbcd715c9c2cdcee2 + # via google-api-core +gviz-api==1.10.0 \ + --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ + --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 + # via xprof +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 + # via + # requests + # yarl +jaraco-functools==4.6.0 \ + --hash=sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280 \ + --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 + # via cheroot +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via werkzeug +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + # via + # cheroot + # jaraco-functools +multidict==6.8.0 \ + --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ + --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ + --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ + --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ + --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ + --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ + --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ + --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ + --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ + --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ + --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ + --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ + --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ + --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ + --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ + --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ + --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ + --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ + --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ + --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ + --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ + --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ + --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ + --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ + --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ + --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ + --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ + --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ + --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ + --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ + --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ + --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ + --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ + --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ + --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ + --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ + --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ + --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ + --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ + --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ + --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ + --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ + --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ + --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ + --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ + --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ + --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ + --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ + --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ + --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ + --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ + --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ + --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ + --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ + --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ + --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ + --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ + --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ + --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ + --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ + --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ + --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ + --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ + --hash=sha256:5cd4637ce76312ba1e05eb9c5193fec231f64fee0944e135fa1e951242355b37 \ + --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ + --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ + --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ + --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ + --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ + --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ + --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ + --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ + --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ + --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ + --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ + --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ + --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ + --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ + --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ + --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ + --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ + --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ + --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ + --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ + --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ + --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ + --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ + --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ + --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ + --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ + --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ + --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ + --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ + --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ + --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ + --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ + --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ + --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ + --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ + --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ + --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ + --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ + --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ + --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ + --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ + --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ + --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ + --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ + --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ + --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ + --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ + --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ + --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ + --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ + --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ + --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ + --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ + --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ + --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ + --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ + --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ + --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ + --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ + --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ + --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ + --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ + --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ + --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ + --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ + --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ + --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ + --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ + --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ + --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ + --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ + --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ + --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ + --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ + --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ + --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ + --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ + --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ + --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ + --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ + --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ + --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ + --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ + --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ + --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ + --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ + --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ + --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ + --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ + --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ + --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ + --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ + --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ + --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ + --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ + --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ + --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ + --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ + --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ + --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ + --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ + --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ + --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ + --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ + --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ + --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ + --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c + # via + # aiohttp + # yarl +oauthlib==3.3.1 \ + --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ + --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 + # via requests-oauthlib +opentelemetry-api==1.44.0 \ + --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ + --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef + # via google-api-core +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 + # via + # aiohttp + # yarl +proto-plus==1.28.4 \ + --hash=sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa \ + --hash=sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63 + # via + # google-api-core + # google-cloud-storage-control +protobuf==7.36.1 \ + --hash=sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00 \ + --hash=sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44 \ + --hash=sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca \ + --hash=sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969 \ + --hash=sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363 \ + --hash=sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c \ + --hash=sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa \ + --hash=sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524 + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus + # xprof +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b + # via pyasn1-modules +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # gcsfs + # google-api-core + # google-cloud-storage + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via google-auth-oauthlib +setuptools==69.5.1 \ + --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ + --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 + # via xprof +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # gviz-api + # xprof +termcolor==3.3.0 \ + --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 + # via fire +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # aiohttp + # aiosignal + # etils + # grpcio + # opentelemetry-api +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # requests + # xprof +werkzeug==3.1.8 \ + --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ + --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 + # via xprof +xprof==2.23.2 \ + --hash=sha256:38e809a29b8226183143d7adf59321f96f6b09ce5a0db0810162a2d9742187fa \ + --hash=sha256:921bc9b1d28ea78d7c63614e28b88dadbe535cf5b6c07c7b65a829a329635a9e \ + --hash=sha256:cdd3387cc357f08713df10b1032977d01c478690e7601f833de13a8ef763aa7b \ + --hash=sha256:d2b802594cf853c509e502a63403c481e607630a98d9ace4cb5fb275c707b5f5 + # via -r tools/xprof/requirements.in +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 + # via aiohttp +zipp==4.1.0 \ + --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ + --hash=sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602 + # via etils diff --git a/tools/xspace_to_perfetto/BUILD.bazel b/tools/xspace_to_perfetto/BUILD.bazel new file mode 100644 index 00000000..5de164d6 --- /dev/null +++ b/tools/xspace_to_perfetto/BUILD.bazel @@ -0,0 +1,32 @@ +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +upb_c_proto_library( + name = "xplane_upb", + deps = ["@tsl//tsl/profiler/protobuf:xplane_proto"], + tags = ["manual"], +) + +zig_library( + name = "xspace_to_perfetto_lib", + tags = ["manual"], + import_name = "tools/xspace_to_perfetto", + main = "xspace_to_perfetto.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xplane_upb", + "//upb", + ], +) + +zig_binary( + name = "xspace_to_perfetto", + main = "cli.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xspace_to_perfetto_lib", + "//stdx", + ], +) diff --git a/tools/xspace_to_perfetto/cli.zig b/tools/xspace_to_perfetto/cli.zig new file mode 100644 index 00000000..f338855a --- /dev/null +++ b/tools/xspace_to_perfetto/cli.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var upb_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_arena.deinit(); + + const args = stdx.flags.parseProcessArgs(init.minimal, xspace_to_perfetto.Args); + const input_path = args.positional.input; + const output_path = args.positional.output orelse try std.fmt.allocPrint(allocator, "{s}.trace.json", .{input_path}); + defer if (args.positional.output == null) allocator.free(output_path); + + const stat = try std.Io.Dir.statFile(.cwd(), io, input_path, .{}); + + var file = try std.Io.Dir.openFile(.cwd(), io, input_path, .{ .mode = .read_only }); + defer file.close(io); + + const xspace = try allocator.alloc(u8, stat.size); + defer allocator.free(xspace); + _ = try file.readPositionalAll(io, xspace, 0); + + try xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile(allocator, io, xspace, output_path); + + var stdout = std.Io.File.stdout().writer(io, &.{}); + try stdout.interface.print("{s}\n", .{output_path}); +} diff --git a/tools/xspace_to_perfetto/xspace_to_perfetto.zig b/tools/xspace_to_perfetto/xspace_to_perfetto.zig new file mode 100644 index 00000000..46aec099 --- /dev/null +++ b/tools/xspace_to_perfetto/xspace_to_perfetto.zig @@ -0,0 +1,712 @@ +const std = @import("std"); + +const c = @import("c"); +const upb = @import("upb"); + +pub const Args = struct { + positional: struct { + input: []const u8, + output: ?[]const u8 = null, + }, + + pub const help = + \\Usage: xspace_to_perfetto [output_trace.json] + \\ + \\Converts an XSpace protobuf profile into trace-viewer JSON. + \\ + ; +}; + +// XPlane schema source: +// ../../xla/third_party/tsl/tsl/profiler/protobuf/xplane.proto +const kHostThreadsPlaneName = "/host:CPU"; +const kGpuPlanePrefix = "/device:GPU:"; +const kTpuPlanePrefix = "/device:TPU:"; +const kCustomPlanePrefix = "/device:CUSTOM:"; +const kXlaAsyncOpLineName = "Async XLA Ops"; +const kOpaqueBytesText = ""; + +const kPicosecondsPerNanosecond: u64 = 1_000; +const kPicosecondsPerMicrosecond: f64 = 1_000_000.0; +const kOutputBufferSizeBytes = 1024 * 1024; +const kHexDigits = "0123456789abcdef"; + +const kFirstDeviceId: u32 = 1; +const kLastDeviceId: u32 = 500; +const kFirstCustomPlaneDeviceId: u32 = kLastDeviceId + 1; +const kMaxCustomPlaneDevicesPerHost: u32 = 200; +const kLastCustomPlaneDeviceId: u32 = kFirstCustomPlaneDeviceId + kMaxCustomPlaneDevicesPerHost - 1; +const kHostThreadsDeviceId: u32 = kLastCustomPlaneDeviceId + 1; + +const StringValue = union(enum) { + text: []const u8, + int64: i64, + uint64: u64, + double: f64, + opaque_bytes, +}; + +const Arg = struct { + name: []const u8, + value: StringValue, +}; + +const StatBehavior = enum { + emit, + step_name, + skip, +}; + +const JsonArrayWriter = struct { + needs_comma: bool = false, + + fn nextItem(self: *JsonArrayWriter, writer: *std.Io.Writer) !void { + if (self.needs_comma) { + try writer.writeByte(','); + } + self.needs_comma = true; + } +}; + +const EventScratch = struct { + args: std.ArrayListUnmanaged(Arg) = .empty, + + fn reset(self: *EventScratch) void { + self.args.clearRetainingCapacity(); + } + + fn deinit(self: *EventScratch, allocator: std.mem.Allocator) void { + self.args.deinit(allocator); + } +}; + +const PlaneContext = struct { + plane: *const c.tensorflow_profiler_XPlane, + device_id: u32, + is_host: bool, + event_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XEventMetadata) = .{}, + stat_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XStatMetadata) = .{}, + event_is_internal_by_id: std.AutoHashMapUnmanaged(i64, bool) = .{}, + stat_behavior_by_id: std.AutoHashMapUnmanaged(i64, StatBehavior) = .{}, + + // Metadata caching mirrors the role of XPlaneVisitor lookups in: + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.h + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + fn init(plane: *const c.tensorflow_profiler_XPlane, device_id: u32, is_host: bool) PlaneContext { + return .{ + .plane = plane, + .device_id = device_id, + .is_host = is_host, + }; + } + + fn deinit(self: *PlaneContext, allocator: std.mem.Allocator) void { + self.event_metadata_by_id.deinit(allocator); + self.stat_metadata_by_id.deinit(allocator); + self.event_is_internal_by_id.deinit(allocator); + self.stat_behavior_by_id.deinit(allocator); + } + + fn getEventMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XEventMetadata { + if (self.event_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XEventMetadata = null; + if (!c.tensorflow_profiler_XPlane_event_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XEventMetadata = resolved; + try self.event_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn getStatMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XStatMetadata { + if (self.stat_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XStatMetadata = null; + if (!c.tensorflow_profiler_XPlane_stat_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XStatMetadata = resolved; + try self.stat_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn isInternalEvent( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !bool { + if (self.event_is_internal_by_id.get(metadata_id)) |is_internal| { + return is_internal; + } + + const is_internal = if (try self.getEventMetadata(allocator, metadata_id)) |metadata| + isInternalEventName(stringView(c.tensorflow_profiler_XEventMetadata_name(metadata))) + else + false; + try self.event_is_internal_by_id.put(allocator, metadata_id, is_internal); + return is_internal; + } + + fn statBehavior( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !StatBehavior { + if (self.stat_behavior_by_id.get(metadata_id)) |behavior| { + return behavior; + } + + const behavior = if (try self.getStatMetadata(allocator, metadata_id)) |metadata| + classifyStatName(stringView(c.tensorflow_profiler_XStatMetadata_name(metadata))) + else + .emit; + try self.stat_behavior_by_id.put(allocator, metadata_id, behavior); + return behavior; + } +}; + +pub fn dumpXSpaceProtoToTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace_proto: []const u8, + output_path: []const u8, +) !void { + var upb_backing_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_backing_arena.deinit(); + + var upb_allocator: upb.Allocator = .init(upb_backing_arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_allocator.inner()); + defer c.upb_Arena_Free(upb_arena); + + const xspace = c.tensorflow_profiler_XSpace_parse(xspace_proto.ptr, xspace_proto.len, upb_arena) orelse + return error.MalformedXSpace; + try writeTraceJsonFile(allocator, io, xspace, output_path); +} + +fn writeTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace: *const c.tensorflow_profiler_XSpace, + output_path: []const u8, +) !void { + var output_file = try std.Io.Dir.createFile(.cwd(), io, output_path, .{}); + defer output_file.close(io); + + const output_buffer = try allocator.alloc(u8, kOutputBufferSizeBytes); + defer allocator.free(output_buffer); + + var file_writer = output_file.writer(io, output_buffer); + defer file_writer.interface.flush() catch {}; + + try streamXSpaceToTraceJson(allocator, xspace, &file_writer.interface); + try file_writer.interface.flush(); +} + +fn streamXSpaceToTraceJson( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, + writer: *std.Io.Writer, +) !void { + // Keep the same trace JSON envelope as the streaming C++ implementation in: + // ../../xla/xla/tsl/profiler/convert/trace_events_to_json.cc + var planes = try collectSelectedPlanes(allocator, xspace); + defer { + for (planes.items) |*plane| { + plane.deinit(allocator); + } + planes.deinit(allocator); + } + + try writer.writeAll("{\"displayTimeUnit\":\"ns\",\"metadata\":{\"highres-ticks\":true},\"traceEvents\":["); + var array_writer: JsonArrayWriter = .{}; + + for (planes.items) |*plane| { + try streamPlaneMetadata(writer, &array_writer, plane); + } + for (planes.items) |*plane| { + try streamPlaneEvents(allocator, writer, &array_writer, plane); + } + + try writer.writeAll("]}"); +} + +fn collectSelectedPlanes( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, +) !std.ArrayListUnmanaged(PlaneContext) { + // Plane selection intentionally follows: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + var result: std.ArrayListUnmanaged(PlaneContext) = .empty; + errdefer { + for (result.items) |*plane| { + plane.deinit(allocator); + } + result.deinit(allocator); + } + + var plane_count: usize = 0; + const planes = c.tensorflow_profiler_XSpace_planes(xspace, &plane_count) orelse return result; + const plane_slice = planes[0..plane_count]; + + var host_plane: ?*const c.tensorflow_profiler_XPlane = null; + for (plane_slice) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + if (std.mem.eql(u8, stringView(c.tensorflow_profiler_XPlane_name(plane)), kHostThreadsPlaneName)) { + host_plane = plane; + break; + } + } + + if (host_plane) |plane| { + try result.append(allocator, PlaneContext.init(plane, kHostThreadsDeviceId, true)); + } + + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kGpuPlanePrefix)) { + return result; + } + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kTpuPlanePrefix)) { + return result; + } + _ = try appendPlanesWithPrefix(allocator, &result, plane_slice, kCustomPlanePrefix); + return result; +} + +fn appendPlanesWithPrefix( + allocator: std.mem.Allocator, + result: *std.ArrayListUnmanaged(PlaneContext), + planes: []const [*c]const c.tensorflow_profiler_XPlane, + prefix: []const u8, +) !bool { + var found = false; + for (planes) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + const name = stringView(c.tensorflow_profiler_XPlane_name(plane)); + if (!std.mem.startsWith(u8, name, prefix)) continue; + found = true; + const device_ordinal = wrapI64ToU32(c.tensorflow_profiler_XPlane_id(plane)); + try result.append(allocator, PlaneContext.init(plane, kFirstDeviceId + device_ordinal, false)); + } + return found; +} + +fn streamPlaneMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *const PlaneContext, +) !void { + // Metadata event shape matches the resource/process metadata emitted in: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const plane_name = stringView(c.tensorflow_profiler_XPlane_name(plane.plane)); + if (plane_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_name\",\"args\":{\"name\":"); + try writeJsonString(writer, plane_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll("}}"); + + var ordinal: u32 = 0; + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + const resource_id = try lineResourceId(line); + const resource_name = lineDisplayName(line); + const sort_index = if (plane.is_host) blk: { + ordinal += 1; + break :blk ordinal; + } else resource_id; + try streamResourceMetadata(writer, array_writer, plane.device_id, resource_id, resource_name, sort_index); + } +} + +fn streamResourceMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + device_id: u32, + resource_id: u32, + resource_name: []const u8, + sort_index: u32, +) !void { + if (resource_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_name\",\"args\":{\"name\":"); + try writeJsonString(writer, resource_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{sort_index}); + try writer.writeAll("}}"); +} + +fn streamPlaneEvents( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, +) !void { + var scratch: EventScratch = .{}; + defer scratch.deinit(allocator); + + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + if (std.mem.eql(u8, lineDisplayName(line), kXlaAsyncOpLineName)) continue; + + const resource_id = try lineResourceId(line); + var event_count: usize = 0; + const events = c.tensorflow_profiler_XLine_events(line, &event_count) orelse continue; + for (events[0..event_count]) |event_c| { + const event: *const c.tensorflow_profiler_XEvent = event_c orelse continue; + try streamEvent(allocator, writer, array_writer, plane, line, event, resource_id, &scratch); + } + } +} + +fn streamEvent( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, + line: *const c.tensorflow_profiler_XLine, + event: *const c.tensorflow_profiler_XEvent, + resource_id: u32, + scratch: *EventScratch, +) !void { + const metadata_id = c.tensorflow_profiler_XEvent_metadata_id(event); + if (try plane.isInternalEvent(allocator, metadata_id)) return; + + // Display name, long_name, stat merge order, and step_name override mirror: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const metadata = try plane.getEventMetadata(allocator, metadata_id); + const metadata_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_name(value)) else ""; + const metadata_display_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_display_name(value)) else ""; + + var event_name: StringValue = .{ + .text = if (metadata_display_name.len != 0) metadata_display_name else metadata_name, + }; + + scratch.reset(); + const args = &scratch.args; + + if (metadata_display_name.len != 0) { + try putArg(args, allocator, "long_name", .{ .text = metadata_name }); + } + + if (metadata) |value| { + try appendStats(allocator, plane, args, &event_name, metadataStats(value)); + } + try appendStats(allocator, plane, args, &event_name, eventStats(event)); + + std.mem.sort(Arg, args.items, {}, argLessThan); + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"X\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"ts\":"); + var number_buf: [64]u8 = undefined; + try writer.writeAll(formatPicosToMicros(&number_buf, eventTimestampPs(line, event))); + try writer.writeAll(",\"dur\":"); + try writer.writeAll(formatPicosToMicros(&number_buf, eventDurationPs(event))); + try writer.writeAll(",\"name\":"); + try writeJsonStringValue(writer, event_name); + + if (args.items.len != 0) { + try writer.writeAll(",\"args\":{"); + for (args.items, 0..) |arg, index| { + if (index != 0) try writer.writeByte(','); + try writeJsonString(writer, arg.name); + try writer.writeByte(':'); + try writeJsonStringValue(writer, arg.value); + } + try writer.writeByte('}'); + } + + try writer.writeByte('}'); +} + +fn appendStats( + allocator: std.mem.Allocator, + plane: *PlaneContext, + args: *std.ArrayListUnmanaged(Arg), + event_name: *StringValue, + stats: []const [*c]const c.tensorflow_profiler_XStat, +) !void { + // Stat filtering and name lookup correspond to: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + for (stats) |stat_c| { + const stat: *const c.tensorflow_profiler_XStat = stat_c orelse continue; + const value_case = c.tensorflow_profiler_XStat_value_case(stat); + if (value_case == c.tensorflow_profiler_XStat_value_NOT_SET) continue; + + const metadata_id = c.tensorflow_profiler_XStat_metadata_id(stat); + const behavior = try plane.statBehavior(allocator, metadata_id); + if (behavior == .skip) continue; + + const stat_value = try stringValueFromStat(allocator, plane, stat); + if (behavior == .step_name) { + event_name.* = stat_value; + } + + const stat_name = if (try plane.getStatMetadata(allocator, metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + try putArg(args, allocator, stat_name, stat_value); + } +} + +fn putArg( + args: *std.ArrayListUnmanaged(Arg), + allocator: std.mem.Allocator, + name: []const u8, + value: StringValue, +) !void { + for (args.items) |*arg| { + if (std.mem.eql(u8, arg.name, name)) { + arg.value = value; + return; + } + } + try args.append(allocator, .{ .name = name, .value = value }); +} + +fn metadataStats(metadata: *const c.tensorflow_profiler_XEventMetadata) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEventMetadata_stats(metadata, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn eventStats(event: *const c.tensorflow_profiler_XEvent) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEvent_stats(event, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn stringValueFromStat( + allocator: std.mem.Allocator, + plane: *PlaneContext, + stat: *const c.tensorflow_profiler_XStat, +) !StringValue { + // Stringification intentionally matches XStatVisitor::ToString(): + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + return switch (c.tensorflow_profiler_XStat_value_case(stat)) { + c.tensorflow_profiler_XStat_value_int64_value => .{ .int64 = c.tensorflow_profiler_XStat_int64_value(stat) }, + c.tensorflow_profiler_XStat_value_uint64_value => .{ .uint64 = c.tensorflow_profiler_XStat_uint64_value(stat) }, + c.tensorflow_profiler_XStat_value_double_value => .{ .double = c.tensorflow_profiler_XStat_double_value(stat) }, + c.tensorflow_profiler_XStat_value_str_value => .{ .text = stringView(c.tensorflow_profiler_XStat_str_value(stat)) }, + c.tensorflow_profiler_XStat_value_bytes_value => .opaque_bytes, + c.tensorflow_profiler_XStat_value_ref_value => blk: { + const ref_metadata_id = std.math.cast(i64, c.tensorflow_profiler_XStat_ref_value(stat)) orelse return error.InvalidStatMetadataId; + const text = if (try plane.getStatMetadata(allocator, ref_metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + break :blk .{ .text = text }; + }, + else => unreachable, + }; +} + +fn writeJsonStringValue(writer: *std.Io.Writer, value: StringValue) !void { + switch (value) { + .text => |text| try writeJsonString(writer, text), + .opaque_bytes => try writeJsonString(writer, kOpaqueBytesText), + .int64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .uint64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .double => |number| { + var buffer: [64]u8 = undefined; + try writeJsonString(writer, formatDouble(&buffer, "%g", number)); + }, + } +} + +fn writeJsonString(writer: *std.Io.Writer, text: []const u8) !void { + try writer.writeByte('"'); + + var chunk_start: usize = 0; + for (text, 0..) |byte, index| { + const escape = switch (byte) { + '"' => "\\\"", + '\\' => "\\\\", + '\n' => "\\n", + '\r' => "\\r", + '\t' => "\\t", + 0x08 => "\\b", + 0x0c => "\\f", + else => null, + }; + if (escape) |escaped| { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writer.writeAll(escaped); + chunk_start = index + 1; + continue; + } + if (byte < 0x20) { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writeAsciiControlEscape(writer, byte); + chunk_start = index + 1; + } + } + + if (chunk_start < text.len) { + try writer.writeAll(text[chunk_start..]); + } + + try writer.writeByte('"'); +} + +fn lineResourceId(line: *const c.tensorflow_profiler_XLine) !u32 { + const display_id = c.tensorflow_profiler_XLine_display_id(line); + const effective_id = if (display_id != 0) display_id else c.tensorflow_profiler_XLine_id(line); + return wrapI64ToU32(effective_id); +} + +fn lineDisplayName(line: *const c.tensorflow_profiler_XLine) []const u8 { + const display_name = stringView(c.tensorflow_profiler_XLine_display_name(line)); + if (display_name.len != 0) return display_name; + return stringView(c.tensorflow_profiler_XLine_name(line)); +} + +fn eventTimestampPs(line: *const c.tensorflow_profiler_XLine, event: *const c.tensorflow_profiler_XEvent) u64 { + const line_timestamp_ns = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XLine_timestamp_ns(line)); + const event_offset_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_offset_ps(event)); + return line_timestamp_ns *% kPicosecondsPerNanosecond +% event_offset_ps; +} + +fn eventDurationPs(event: *const c.tensorflow_profiler_XEvent) u64 { + const duration_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_duration_ps(event)); + return @max(duration_ps, 1); +} + +fn formatPicosToMicros(buffer: *[64]u8, picoseconds: u64) []const u8 { + // The conversion and "%.17g" formatting are kept identical to: + // ../../xla/xla/tsl/profiler/utils/math_utils.h + // ../../xla/xla/tsl/profiler/utils/format_utils.h + const microseconds = @as(f64, @floatFromInt(picoseconds)) / kPicosecondsPerMicrosecond; + return formatDouble(buffer, "%.17g", microseconds); +} + +fn formatDouble(buffer: *[64]u8, comptime fmt: [*:0]const u8, value: f64) []const u8 { + const written = c.snprintf(buffer.ptr, buffer.len, fmt, value); + if (written <= 0 or written >= buffer.len) unreachable; + return buffer[0..@intCast(written)]; +} + +fn stringView(value: c.upb_StringView) []const u8 { + const text = upb.slice(value); + return text orelse ""; +} + +fn reinterpretSignedAsUnsigned64(value: i64) u64 { + const same_bits: u64 = @bitCast(value); + return same_bits; +} + +fn wrapI64ToU32(value: i64) u32 { + const unsigned_value = reinterpretSignedAsUnsigned64(value); + return @truncate(unsigned_value); +} + +fn writeAsciiControlEscape(writer: *std.Io.Writer, byte: u8) !void { + const high_nibble = @as(usize, byte / 16); + const low_nibble = @as(usize, byte % 16); + + try writer.writeAll("\\u00"); + try writer.writeByte(kHexDigits[high_nibble]); + try writer.writeByte(kHexDigits[low_nibble]); +} + +fn argLessThan(_: void, lhs: Arg, rhs: Arg) bool { + return std.mem.order(u8, lhs.name, rhs.name) == .lt; +} + +fn isInternalEventName(name: []const u8) bool { + // Internal host-event filtering is derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + inline for ([_][]const u8{ + "MemoryAllocation", + "MemoryDeallocation", + "PrefetchProduce", + "PrefetchConsume", + "ParallelInterleaveProduce", + "ParallelInterleaveConsume", + "ParallelInterleaveInitializeInput", + "ParallelMapProduce", + "ParallelMapConsume", + "MapAndBatchProduce", + "MapAndBatchConsume", + "ParseExampleProduce", + "ParseExampleConsume", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return true; + } + return false; +} + +fn classifyStatName(name: []const u8) StatBehavior { + // Internal stat filtering and step_name handling are derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + if (std.mem.eql(u8, name, "step_name")) return .step_name; + inline for ([_][]const u8{ + "_pt", + "_ct", + "_p", + "_c", + "_r", + "flops", + "program_id", + "symbol_id", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return .skip; + } + return .emit; +} diff --git a/tools/zls.sh b/tools/zls.sh index 4b59ccaf..2b83f7b6 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,7 +1,4 @@ #!/usr/bin/env bash - -set -euo pipefail - cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(bazel info workspace)" exec bazel run -- @zml//:completion "${@}" diff --git a/translate-c/BUILD.bazel b/translate-c/BUILD.bazel deleted file mode 100644 index 91f3fc51..00000000 --- a/translate-c/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -alias( - name = "translate-c", - actual = "@translate-c", - visibility = ["//visibility:public"], -) - -alias( - name = "helpers", - actual = "@translate-c//:helpers", - visibility = ["//visibility:public"], -) - -alias( - name = "c_builtins", - actual = "@translate-c//:c_builtins", - visibility = ["//visibility:public"], -) diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index ebf973fb..9189fc8f 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "upb_c", + tags = ["manual"], hdrs = ["upb.h"], deps = [ "@com_google_protobuf//upb/message:copy", @@ -11,6 +12,7 @@ cc_library( zig_library( name = "upb", + tags = ["manual"], main = "upb.zig", visibility = ["//visibility:public"], deps = [":upb_c"], diff --git a/upb/upb.zig b/upb/upb.zig index b58252d7..f07d003a 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -34,7 +34,6 @@ pub const ParseError = error{ BadUtf8, MaxDepthExceeded, MissingRequired, - UnlinkedSubMessage, Unknown, } || std.mem.Allocator.Error; @@ -102,7 +101,6 @@ pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, op c.kUpb_DecodeStatus_BadUtf8 => ParseError.BadUtf8, c.kUpb_DecodeStatus_MaxDepthExceeded => ParseError.MaxDepthExceeded, c.kUpb_DecodeStatus_MissingRequired => ParseError.MissingRequired, - c.kUpb_DecodeStatus_UnlinkedSubMessage => ParseError.UnlinkedSubMessage, else => ParseError.Unknown, }; } diff --git a/zml/io/vfs/BUILD.bazel b/vfs/BUILD.bazel similarity index 52% rename from zml/io/vfs/BUILD.bazel rename to vfs/BUILD.bazel index 9f79dd24..c8de1e0c 100644 --- a/zml/io/vfs/BUILD.bazel +++ b/vfs/BUILD.bazel @@ -1,17 +1,25 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( name = "vfs", srcs = [ "base.zig", "file.zig", + "gcs.zig", "hf.zig", "http.zig", + "parallel_read.zig", "s3.zig", ], + main = "vfs.zig", + tags = ["manual"], + visibility = ["//visibility:public"], deps = [ "//stdx", ], - main = "index.zig", - visibility = ["//visibility:public"], +) + +zig_test( + name = "test", + deps = [":vfs"], ) diff --git a/zml/io/vfs/base.zig b/vfs/base.zig similarity index 91% rename from zml/io/vfs/base.zig rename to vfs/base.zig index 041d4689..1e66ca89 100644 --- a/zml/io/vfs/base.zig +++ b/vfs/base.zig @@ -23,7 +23,6 @@ pub const VFSBase = struct { .recancel = recancel, .swapCancelProtection = swapCancelProtection, .checkCancel = checkCancel, - .select = select, .futexWait = futexWait, .futexWaitUncancelable = futexWaitUncancelable, .futexWake = futexWake, @@ -92,13 +91,14 @@ pub const VFSBase = struct { .unlockStderr = unlockStderr, .processCurrentPath = processCurrentPath, .processSetCurrentDir = processSetCurrentDir, + .processSetCurrentPath = processSetCurrentPath, .processReplace = processReplace, - .processReplacePath = processReplacePath, .processSpawn = processSpawn, - .processSpawnPath = processSpawnPath, .childWait = childWait, .childKill = childKill, .progressParentFile = progressParentFile, + .inheritParentDir = inheritParentDir, + .inheritParentFile = inheritParentFile, .now = now, .clockResolution = clockResolution, .sleep = sleep, @@ -111,10 +111,6 @@ pub const VFSBase = struct { .netListenUnix = netListenUnix, .netConnectUnix = netConnectUnix, .netSocketCreatePair = netSocketCreatePair, - .netSend = netSend, - .netReceive = netReceive, - .netRead = netRead, - .netWrite = netWrite, .netWriteFile = netWriteFile, .netClose = netClose, .netShutdown = netShutdown, @@ -157,12 +153,12 @@ pub const VFSBase = struct { return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) void { + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) std.Io.ConcurrentError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } @@ -192,11 +188,6 @@ pub const VFSBase = struct { return self.inner.vtable.checkCancel(self.inner.userdata); } - pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.select(self.inner.userdata, futures); - } - pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); @@ -267,7 +258,7 @@ pub const VFSBase = struct { return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); } - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); } @@ -277,7 +268,7 @@ pub const VFSBase = struct { return self.inner.vtable.dirCreateFileAtomic(self.inner.userdata, dir, sub_path, options); } - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); } @@ -502,7 +493,7 @@ pub const VFSBase = struct { return self.inner.vtable.fileMemoryMapWrite(self.inner.userdata, memory_map); } - pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.Dir.OpenFileOptions) std.process.OpenExecutableError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); } @@ -537,26 +528,21 @@ pub const VFSBase = struct { return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } + pub fn processSetCurrentPath(userdata: ?*anyopaque, path: []const u8) std.process.SetCurrentPathError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSetCurrentPath(self.inner.userdata, path); + } + pub fn processReplace(userdata: ?*anyopaque, options: std.process.ReplaceOptions) std.process.ReplaceError { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processReplace(self.inner.userdata, options); } - pub fn processReplacePath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.ReplaceOptions) std.process.ReplaceError { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processReplacePath(self.inner.userdata, dir, options); - } - pub fn processSpawn(userdata: ?*anyopaque, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processSpawn(self.inner.userdata, options); } - pub fn processSpawnPath(userdata: ?*anyopaque, dir: std.Io.Dir, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processSpawnPath(self.inner.userdata, dir, options); - } - pub fn childWait(userdata: ?*anyopaque, child: *std.process.Child) std.process.Child.WaitError!std.process.Child.Term { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.childWait(self.inner.userdata, child); @@ -572,6 +558,16 @@ pub const VFSBase = struct { return self.inner.vtable.progressParentFile(self.inner.userdata); } + pub fn inheritParentDir(userdata: ?*anyopaque, handle: std.Io.Dir.Handle) std.Io.InheritParentHandleError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentDir(self.inner.userdata, handle); + } + + pub fn inheritParentFile(userdata: ?*anyopaque, handle: std.Io.File.Handle, flags: std.Io.File.Flags) std.Io.InheritParentHandleError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentFile(self.inner.userdata, handle, flags); + } + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.now(self.inner.userdata, clock); @@ -597,14 +593,14 @@ pub const VFSBase = struct { return self.inner.vtable.randomSecure(self.inner.userdata, buffer); } - pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { + pub fn netListenIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenIp(self.inner.userdata, address, options); } - pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle, options: std.Io.net.Server.AcceptOptions) std.Io.net.Server.AcceptError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netAccept(self.inner.userdata, server); + return self.inner.vtable.netAccept(self.inner.userdata, server, options); } pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { @@ -612,7 +608,7 @@ pub const VFSBase = struct { return self.inner.vtable.netBindIp(self.inner.userdata, address, options); } - pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Socket { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); } @@ -632,34 +628,14 @@ pub const VFSBase = struct { return self.inner.vtable.netSocketCreatePair(self.inner.userdata, options); } - pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); - } - - pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - - pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netRead(self.inner.userdata, src, data); - } - - pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); - } - pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); } - pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { + pub fn netClose(userdata: ?*anyopaque, sockets: []const std.Io.net.Socket) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.netClose(self.inner.userdata, handles); + self.inner.vtable.netClose(self.inner.userdata, sockets); } pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { diff --git a/zml/io/vfs/file.zig b/vfs/file.zig similarity index 71% rename from zml/io/vfs/file.zig rename to vfs/file.zig index ff8df678..1b077c33 100644 --- a/zml/io/vfs/file.zig +++ b/vfs/file.zig @@ -5,7 +5,7 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/file"); +const log = std.log.scoped(.@"zml/vfs/file"); pub const DirectIoError = error{ UnsupportedPlatform, @@ -21,9 +21,30 @@ fn canUseDirectIO() bool { return false; } +fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { + while (true) { + const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); + switch (std.posix.errno(result)) { + .SUCCESS => return result, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { + while (true) { + switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { + .SUCCESS => return, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -33,12 +54,11 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; - const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags & ~direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -46,11 +66,9 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = std.posix.system.fcntl(file.handle, std.posix.F.GETFL); + const flags = try getStatusFlags(file); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - - const result = std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); - if (result != 0) return DirectIoError.UnexpectedFcntlResult; + try setStatusFlags(file, flags | direct_flag); } else { return DirectIoError.UnsupportedPlatform; } @@ -85,7 +103,7 @@ pub const File = struct { allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, config: Config, base: VFSBase, @@ -111,6 +129,7 @@ pub const File = struct { .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -119,6 +138,9 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -151,7 +173,7 @@ pub const File = struct { return self.handles.at(@intCast(file.handle)); } - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; @@ -201,7 +223,18 @@ pub const File = struct { }, }); }, - .file_write_streaming, .device_io_control => { + .file_write_streaming => |o| { + const handle = self.getFileHandle(o.file); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_write_streaming = .{ + .file = handle.innerFile(), + .header = o.header, + .data = o.data, + .splat = o.splat, + }, + }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -227,7 +260,21 @@ pub const File = struct { return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = null, + }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); errdefer inner_file.close(self.base.inner); @@ -311,6 +358,52 @@ pub const File = struct { } } + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + } + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = self.innerFile(file, data, offset); diff --git a/vfs/gcs.zig b/vfs/gcs.zig new file mode 100644 index 00000000..acc7f634 --- /dev/null +++ b/vfs/gcs.zig @@ -0,0 +1,1138 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/gcs"); + +const EndpointUrl = "https://storage.googleapis.com"; +const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; + +/// Check if running on Google Compute Engine / Cloud Run host infrastructure, +/// because metadata server probing should only happen on GCP. +/// Mirrors the TPU platform detection approach. +fn isOnGCP(io: std.Io) !bool { + if (builtin.os.tag != .linux) return false; + + const GoogleComputeEngine = "Google Compute Engine"; + var buffer: [GoogleComputeEngine.len]u8 = undefined; + return std.mem.eql( + u8, + GoogleComputeEngine, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); +} + +const OAuthToken = struct { + // https://developers.google.com/identity/protocols/oauth2#size + const Size = 2048; + + access_token: []const u8, + expires_in: u64, +}; + +const Credentials = union(enum) { + const AuthorizedUser = struct { + type: []const u8, + client_id: []const u8, + client_secret: []const u8, + refresh_token: []const u8, + quota_project_id: ?[]const u8 = null, + account: []const u8, + universe_domain: []const u8, + }; + + const ServiceAccount = struct { + pub const JwtAssertion = struct { + iss: []const u8, + scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", + aud: []const u8 = "https://oauth2.googleapis.com/token", + exp: i64, + iat: i64, + sub: ?[]const u8 = null, + }; + + type: []const u8, + client_email: []const u8, + private_key: []const u8, + token_uri: []const u8, + quota_project_id: ?[]const u8 = null, + + pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { + const now: std.Io.Timestamp = .now(io_, .real); + return .{ + .iss = self.client_email, + .iat = now.toSeconds(), + .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), + }; + } + }; + + authorized_user: AuthorizedUser, + service_account: ServiceAccount, + metadata_server: void, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Credentials { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Credentials { + return switch (source) { + .object => |obj| blk: { + const obj_type = obj.get("type") orelse return std.json.ParseFromValueError.UnexpectedToken; + if (std.ascii.eqlIgnoreCase(obj_type.string, "authorized_user")) { + break :blk .{ + .authorized_user = try std.json.parseFromValueLeaky( + AuthorizedUser, + allocator, + source, + options, + ), + }; + } else if (std.ascii.eqlIgnoreCase(obj_type.string, "service_account")) { + break :blk .{ + .service_account = try std.json.parseFromValueLeaky( + ServiceAccount, + allocator, + source, + options, + ), + }; + } + log.warn("Unknown credentials type: {s}", .{obj_type.string}); + return std.json.ParseFromValueError.InvalidEnumTag; + }, + else => std.json.ParseFromValueError.UnexpectedToken, + }; + } +}; + +const ReadState = struct { index: usize, objects: [][]const u8 }; + +pub const GCS = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *GCS, + uri: std.Uri, + path: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request( + .GET, + job.batch.uri, + .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }, + ) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; + + const Token = struct { + header: []u8, + expires_at: std.Io.Timestamp, + + fn expired(self: *const Token, io_: std.Io) bool { + const now: std.Io.Timestamp = .now(io_, .real); + return now.toSeconds() >= self.expires_at.toSeconds(); + } + }; + + pub const Config = struct { + credentials: ?Credentials = null, + endpoint_uri: std.Uri, + region: []const u8 = "auto", + }; + + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + return .{ + .type = type_, + .uri = try allocator.dupe(u8, path), + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + config: Config, + token: Token, + read_pool: *ParallelRead.Pool, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, + base: VFSBase, + + pub const InitOpts = struct { + credentials: ?union(enum) { + json: *std.Io.Reader, + metadata_server: void, + } = null, + endpoint_url: []const u8 = "https://storage.googleapis.com", + region: []const u8 = "auto", + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + + pub const InitError = error{ + InvalidCredentialJson, + Unexpected, + } || std.mem.Allocator.Error || std.Io.ConcurrentError; + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + const config: Config = .{ + .credentials = if (opts.credentials) |creds| switch (creds) { + .json => |reader| blk: { + var json_reader: std.json.Reader = .init(allocator, reader); + defer json_reader.deinit(); + + break :blk std.json.parseFromTokenSourceLeaky( + Credentials, + arena.allocator(), + &json_reader, + .{ .allocate = .alloc_always }, + ) catch return InitError.InvalidCredentialJson; + }, + .metadata_server => .{ .metadata_server = {} }, + } else null, + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, + .region = try arena.allocator().dupe(u8, opts.region), + }; + + const token: Token = .{ + .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .expires_at = .zero, + }; + + return .{ + .allocator = allocator, + .arena = arena, + .base = .init(inner), + .client = http_client, + .config = config, + .token = token, + .read_pool = read_pool, + }; + } + + pub fn auto(allocator: std.mem.Allocator, inner_io: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !GCS { + var jsonBuffer: [1024]u8 = undefined; + + const credentials_file: ?std.Io.File = if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| blk: { + break :blk std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}) catch |err| { + log.err("Unable to open GCS credentials file {s}, skipping: {t}", .{ json_path, err }); + break :blk null; + }; + } else if (applicationDefaultCredentials(inner_io, environ_map)) |f| + f + else + null; + defer if (credentials_file) |f| f.close(inner_io); + + if (credentials_file) |f| blk: { + var reader = f.reader(inner_io, &jsonBuffer); + return GCS.init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }) catch |err| switch (err) { + InitError.InvalidCredentialJson => { + log.err("Invalid GCS credentials JSON", .{}); + break :blk; + }, + else => return err, + }; + } + + if (isOnGCP(inner_io) catch false) { + return .init(allocator, inner_io, http_client, .{ .credentials = .{ .metadata_server = {} } }); + } + + return .init(allocator, inner_io, http_client, .{}); + } + + fn applicationDefaultCredentials(io_: std.Io, environ_map: *std.process.Environ.Map) ?std.Io.File { + var buffer: [1024]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoin(&buffer, switch (builtin.os.tag) { + .windows => &.{ environ_map.get("APPDATA") orelse return null, "gcloud", "application_default_credentials.json" }, + else => &.{ environ_map.get("HOME") orelse return null, ".config", "gcloud", "application_default_credentials.json" }, + }) catch unreachable; + return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; + } + + fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = MetadataUrl }, + .method = .GET, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + }, + .extra_headers = &.{ + .{ .name = "Metadata-Flavor", .value = "Google" }, + }, + .response_writer = &response_writer, + }); + + if (result.status != .ok) { + return null; + } + + return response_writer.buffered(); + } + + fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = "https://oauth2.googleapis.com/token" }, + .method = .POST, + .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }), + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }, + .response_writer = &response_writer, + }); + if (result.status != .ok) { + log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + return error.RequestFailed; + } + return response_writer.buffered(); + } + + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { + _ = io_; + _ = client; + _ = service_account; + _ = buffer; + return error.Unimplemented; + + // const assertion = service_account.makeAssertion(io_); + + // var tmp_buffer: [8 * 1024]u8 = undefined; + + // var buffer_writer: std.Io.Writer = .fixed(buffer); + // var signing_input_buffer: [2 * 1024]u8 = undefined; + // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); + + // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; + // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); + // std.base64.url_safe_no_pad.Encoder.encodeWriter( + // signing_input_writer, + // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ + // std.json.fmt(assertion, .{}), + // }), + // ); + // const signing_input = signing_input_writer.buffered(); + // _ = signing_input; + + // const result = try client.fetch(.{ + // .location = .{ .url = "https://oauth2.googleapis.com/token" }, + // .method = .POST, + // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + // }), + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .content_type = .{ .override = "application/x-www-form-urlencoded" }, + // }, + // .response_writer = &response_writer, + // }); + // if (result.status != .ok) { + // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + // return error.RequestFailed; + // } + } + + fn refreshToken(self: *GCS) !void { + var buffer: [4 * 1024]u8 = undefined; + const payload = switch (self.config.credentials.?) { + .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), + .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + }; + const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ + .allocate = .alloc_if_needed, + .ignore_unknown_fields = true, + }); + defer oauth_token.deinit(); + self.token = .{ + .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), + .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), + }; + } + + fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + if (self.config.credentials == null) { + return .omit; + } + + if (self.token.expired(self.base.inner)) { + try self.refreshToken(); + } + return .{ .override = self.token.header }; + } + + pub fn deinit(self: *GCS) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.dir_read_states.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.objects) |obj| { + self.allocator.free(obj); + } + self.allocator.free(entry.value_ptr.objects); + } + self.dir_read_states.deinit(self.allocator); + self.arena.deinit(); + } + + pub fn io(self: *GCS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *GCS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *GCS, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *GCS, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *GCS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *GCS, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = error.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(@intFromPtr(handle)), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + if (self.dir_read_states.fetchRemove(reader)) |kv| { + for (kv.value.objects) |obj| self.allocator.free(obj); + self.allocator.free(kv.value.objects); + } + + const handle = self.getDirHandle(reader.dir); + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + + self.dir_read_states.put(self.allocator, reader, .{ + .index = 0, + .objects = objects, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (objects.len > 0) .reading else .finished; + + if (objects.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.objects.len) { + const obj_key = state.objects[state.index]; + const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + + const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + obj_key[idx + 1 ..] + else + obj_key; + + entries[count] = .{ + .name = std.mem.trimEnd(u8, name, "/"), + .kind = kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.objects.len) { + reader.state = .finished; + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @backingInt(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { + const path = std.mem.trim(u8, path_, "/"); + + if (std.mem.indexOfScalar(u8, path, '/')) |idx| { + return .{ path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + } else { + return .{ path, "" }; + } + } + + fn gcsUri(self: *GCS, path: []const u8) std.Uri { + var uri = self.config.endpoint_uri; + uri.path = .{ .raw = path }; + uri.query = .{ .percent_encoded = "alt=media" }; + return uri; + } + + // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { + // authorization: ?[]const u8, + // extra_headers: [3]?std.http.Header, + // extra_len: usize, + // } { + // if (self.config.oauth_access_token == null) { + // if (self.config.googleCredentialsJsonStr) |json| { + // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); + // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); + // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); + // self.config.oauth_access_token = oauth.access_token; + // self.config.quota_project_id = oauth.quota_project_id; + // } + // } + + // const token = self.config.oauth_access_token orelse return error.MissingCredentials; + // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); + // var extra_headers: [3]?std.http.Header = .{ null, null, null }; + // var extra_len: usize = 0; + // if (self.config.quota_project_id) |project| { + // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; + // extra_len = 1; + // } + // return .{ + // .authorization = authorization, + // .extra_headers = extra_headers, + // .extra_len = extra_len, + // }; + // } + + fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { + var query_buf: [4096]u8 = undefined; + var query_writer = std.Io.Writer.fixed(&query_buf); + + try query_writer.writeAll("delimiter="); + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + try query_writer.writeAll("&max-keys=1000"); + + if (key_prefix.len > 0) { + try query_writer.writeAll("&prefix="); + try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + } + } + + var path_buf: [256]u8 = undefined; + var uri = self.config.endpoint_uri; + uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; + uri.query = .{ .percent_encoded = query_writer.buffered() }; + + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = if (self.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } + + fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { + const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(bucket, key_prefix); + defer self.allocator.free(body); + + return try self.parseListObjectsResponse(body, key_prefix); + } + + fn gcsEncodeIsValid(c: u8) bool { + return switch (c) { + 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, + else => false, + }; + } + + fn quotaProjectId(self: *const GCS) ?[]const u8 { + const credentials = self.config.credentials orelse return null; + return switch (credentials) { + .authorized_user => |authorized_user| authorized_user.quota_project_id, + .service_account => |service_account| service_account.quota_project_id, + .metadata_server => null, + }; + } + + fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { + var results: std.ArrayListUnmanaged([]const u8) = .empty; + errdefer { + for (results.items) |item| self.allocator.free(item); + results.deinit(self.allocator); + } + + inline for (.{ "Key", "Prefix" }) |tag| { + var pos: usize = 0; + while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + const v_start = start + tag.len + 2; + const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + const val = xml[v_start..end]; + + const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + if (val.len > 0 and !is_self) { + try results.append(self.allocator, try self.allocator.dupe(u8, val)); + } + pos = end + tag.len + 3; + } + } + + return try results.toOwnedSlice(self.allocator); + } + + fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const uri = self.gcsUri(path); + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + const size = switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + .unauthorized, .forbidden => return error.PermissionDenied, + else => blk: { + log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + return size; + } + + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; + + const uri = self.gcsUri(handle.uri); + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .path = handle.uri, + .authorization = try self.getOrRefreshToken(), + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; + } + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } +}; + +test "GCS parses XML bucket listing objects and common prefixes" { + const xml = + "" ++ + "" ++ + "bucket" ++ + "LC08/01/001/002/" ++ + "LC08/01/001/002/scene_$folder$6" ++ + "LC08/01/001/002/scene/" ++ + "LC08/01/001/002/" ++ + ""; + + var backend: GCS = undefined; + backend.allocator = std.testing.allocator; + + const objects = try backend.parseListObjectsResponse(xml, "LC08/01/001/002/"); + defer { + for (objects) |object| std.testing.allocator.free(object); + std.testing.allocator.free(objects); + } + + try std.testing.expectEqual(@as(usize, 2), objects.len); + try std.testing.expectEqualStrings("LC08/01/001/002/scene_$folder$", objects[0]); + try std.testing.expectEqualStrings("LC08/01/001/002/scene/", objects[1]); +} + +test "GCS splits bucket and object prefix" { + var backend: GCS = undefined; + + const root_bucket, const root_prefix = backend.pathComponents("gcp-public-data-landsat/"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", root_bucket); + try std.testing.expectEqualStrings("", root_prefix); + + const bucket, const prefix = backend.pathComponents("gcp-public-data-landsat/LC08/01"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); + try std.testing.expectEqualStrings("LC08/01", prefix); +} diff --git a/zml/io/vfs/hf.zig b/vfs/hf.zig similarity index 73% rename from zml/io/vfs/hf.zig rename to vfs/hf.zig index ca9a0bc8..8770fe18 100644 --- a/zml/io/vfs/hf.zig +++ b/vfs/hf.zig @@ -2,9 +2,159 @@ const std = @import("std"); const stdx = @import("stdx"); +const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/hf"); +const log = std.log.scoped(.@"zml/vfs/hf"); + +const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + uri: std.Uri, + url: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + pub const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout => .retry(), + .too_many_requests => b: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const p = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, p, "t=")) { + const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; + break :b .{ .retry_after = .fromSeconds(seconds) }; + } + } + } + } + break :b .retry(); + }, + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; +}; pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; @@ -50,6 +200,18 @@ const RepoKey = struct { }; pub const HF = struct { + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + // Chunk size needs to be big enough to avoid hitting the rate limits of HF. + .chunk_size = 32 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + pub const Repo = struct { repo: []const u8, model: []const u8, @@ -102,14 +264,21 @@ pub const HF = struct { mutex: std.Io.Mutex = .init, client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8) !HF { - return .{ + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + var self: HF = .{ .allocator = allocator, .base = .init(inner), .client = http_client, @@ -120,7 +289,14 @@ pub const HF = struct { } else blk: { break :blk .default; }, + .read_pool = read_pool, }; + errdefer switch (self.authorization) { + .default, .omit => {}, + .override => |t| self.allocator.free(t), + }; + + return self; } pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { @@ -145,10 +321,13 @@ pub const HF = struct { if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token); + return init(allocator, inner, http_client, hf_token, .{}); } pub fn deinit(self: *HF) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -290,9 +469,13 @@ pub const HF = struct { .path = "", }); const uri = try std.Uri.parse(url); + log.info("Fetching from HF: {s}", .{url}); var req = try self.client.request(.GET, uri, .{ - .headers = .{ .authorization = self.authorization }, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = self.authorization, + }, }); defer req.deinit(); @@ -316,7 +499,7 @@ pub const HF = struct { ); defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .{}; + var tree_root: std.ArrayList(TreeNode) = .empty; for (parsed.value) |item| { try insertTreeNode(self.allocator, &tree_root, item); } @@ -404,7 +587,7 @@ pub const HF = struct { var parts = std.mem.tokenizeScalar(u8, item.path, '/'); var current_list = root; - var parents: std.ArrayList(*TreeNode) = .{}; + var parents: std.ArrayList(*TreeNode) = .empty; defer parents.deinit(allocator); while (true) { @@ -423,7 +606,7 @@ pub const HF = struct { if (!is_last) { try parents.append(allocator, f); if (f.children == null) { - f.children = .{}; + f.children = .empty; } current_list = &f.children.?; } else if (std.mem.eql(u8, item.type, "file")) { @@ -435,7 +618,7 @@ pub const HF = struct { .name = try allocator.dupe(u8, part), .kind = if (is_file) .file else .directory, .size = if (is_last) item.size else 0, - .children = if (is_file) null else .{}, + .children = if (is_file) null else .empty, }; try current_list.append(allocator, new_node); @@ -459,17 +642,17 @@ pub const HF = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -549,7 +732,7 @@ pub const HF = struct { if (node == null) return std.Io.Dir.AccessError.FileNotFound; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); var path_buffer: [8 * 1024]u8 = undefined; @@ -664,7 +847,7 @@ pub const HF = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } @@ -694,79 +877,44 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; - }; + var read_size: usize = 0; + for (data) |buf| read_size += buf.len; + read_size = @intCast(@min(handle.size - offset, read_size)); + if (read_size == 0) return 0; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); const repo: Repo = try .parse(handle.uri); var url_buffer: [8 * 1024]u8 = undefined; - const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); - + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); const uri: std.Uri = try .parse(url); - var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }); - defer req.deinit(); - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to perform read for {s}", .{url}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .uri = uri, + .url = url, + .authorization = self.authorization, }; - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - return try reader.readSliceShort(data[0]); - } + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return read_size; } }; diff --git a/zml/io/vfs/http.zig b/vfs/http.zig similarity index 97% rename from zml/io/vfs/http.zig rename to vfs/http.zig index c2bfa332..9054983e 100644 --- a/zml/io/vfs/http.zig +++ b/vfs/http.zig @@ -4,7 +4,7 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/http"); +const log = std.log.scoped(.@"zml/vfs/http"); pub const HTTP = struct { const Handle = struct { @@ -42,7 +42,7 @@ pub const HTTP = struct { client: *std.http.Client, protocol: Protocol, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, base: VFSBase, pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { @@ -146,17 +146,17 @@ pub const HTTP = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -210,7 +210,7 @@ pub const HTTP = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; @@ -284,7 +284,7 @@ pub const HTTP = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig new file mode 100644 index 00000000..70c9d89d --- /dev/null +++ b/vfs/parallel_read.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const InitOpts = struct { + chunk_size: usize, + num_workers: usize, + queue_capacity: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, +}; + +pub const Status = union(enum) { + success, + retry_after: ?std.Io.Duration, + + pub fn retry() Status { + return .{ .retry_after = null }; + } +}; + +pub const BatchState = struct { + pending: std.atomic.Value(u32), + err: std.atomic.Value(u16) = .init(0), + + pub fn failed(batch: *BatchState) bool { + return batch.err.load(.acquire) != 0; + } + + pub fn fail(batch: *BatchState, err: anyerror) void { + _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); + } + + pub fn anyError(batch: *BatchState) ?anyerror { + const err = batch.err.load(.acquire); + return if (err == 0) null else @errorFromInt(err); + } + + pub fn completeOne(batch: *BatchState, io: std.Io) void { + const previous = batch.pending.fetchSub(1, .release); + std.debug.assert(previous > 0); + if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); + } + + pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { + while (true) { + const pending = batch.pending.load(.acquire); + if (pending == 0) return; + io.futexWaitUncancelable(u32, &batch.pending.raw, pending); + } + } +}; + +pub fn Pool(comptime Job: type) type { + return struct { + const Self = @This(); + + group: std.Io.Group = .init, + job_queue: std.Io.Queue(Job), + queue_buf: []Job, + client: *std.http.Client, + prng: std.Random.DefaultPrng, + chunk_size: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, + + pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { + stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); + stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); + stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); + + const queue_buf = try allocator.alloc(Job, opts.queue_capacity); + errdefer allocator.free(queue_buf); + + self.* = .{ + .client = client, + .chunk_size = opts.chunk_size, + .max_retries = opts.max_retries, + .retry_initial_delay = opts.retry_initial_delay, + .retry_max_delay = opts.retry_max_delay, + .prng = .init(blk: { + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + break :blk seed; + }), + .job_queue = .init(queue_buf), + .queue_buf = queue_buf, + }; + errdefer self.group.cancel(io); + + for (0..opts.num_workers) |_| { + try self.group.concurrent(io, worker, .{ self, io }); + } + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { + self.job_queue.close(io); + self.group.cancel(io); + + allocator.free(self.queue_buf); + } + + fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { + while (true) { + const job = pool.job_queue.getOne(io) catch break; + + var attempt: usize = 0; + while (true) { + if (job.batch.state.failed()) break; + const status = job.perform(pool.client) catch |err| { + job.batch.state.fail(err); + break; + }; + + switch (status) { + .success => break, + .retry_after => |duration| { + if (attempt >= pool.max_retries) { + job.batch.state.fail(error.RetriesExhausted); + break; + } + + io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { + job.batch.state.fail(error.RetriesExhausted); + break; + }; + + attempt += 1; + }, + } + continue; + } + + job.batch.state.completeOne(io); + } + } + + fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { + // FullJitter algorithm + const max_delay_ns: i96 = @min( + pool.retry_max_delay.toNanoseconds(), + pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); + return .fromNanoseconds(delay_ns); + } + }; +} + +pub const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +pub fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; +} + +pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested += buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +pub fn readChunk( + reader: *std.Io.Reader, + content_range: ?ContentRange, + file_offset: u64, + data: []const []u8, + chunk_offset: usize, + chunk_len: usize, +) !void { + if (content_range) |cr| { + if (cr.start > file_offset) return error.InvalidContentRange; + if (cr.start < file_offset) { + try reader.discardAll(file_offset - cr.start); + } + } + + var remaining = chunk_len; + var skip = chunk_offset; + + for (data) |buf| { + if (skip >= buf.len) { + skip -= buf.len; + continue; + } + + const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; + try reader.readSliceAll(destination); + remaining -= destination.len; + if (remaining == 0) break; + skip = 0; + } +} diff --git a/zml/io/vfs/s3.zig b/vfs/s3.zig similarity index 72% rename from zml/io/vfs/s3.zig rename to vfs/s3.zig index 193009e0..eb849e21 100644 --- a/zml/io/vfs/s3.zig +++ b/vfs/s3.zig @@ -2,9 +2,10 @@ const std = @import("std"); const stdx = @import("stdx"); +const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/io/vfs/s3"); +const log = std.log.scoped(.@"zml/vfs/s3"); pub const AwsSigV4 = struct { access_key: ?[]const u8, @@ -140,6 +141,177 @@ pub const AwsSigV4 = struct { const ReadState = struct { index: usize, objects: [][]const u8 }; pub const S3 = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *S3, + uri: std.Uri, + url: []const u8, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; + + const signer: AwsSigV4 = .{ + .access_key = job.batch.backend.config.access_key, + .secret_key = job.batch.backend.config.secret_key, + .region = job.batch.backend.config.region, + .service = job.batch.backend.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = signer.generateAuthHeader( + &authorization_buffer, + .GET, + job.batch.uri, + timestamp, + &.{.{ .name = "Range", .value = range_header }}, + ) catch |err| { + log.err("Failed to generate auth header: {}", .{err}); + return err; + }; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; + + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + pub const Config = struct { access_key: ?[]const u8 = null, secret_key: ?[]const u8 = null, @@ -174,8 +346,9 @@ pub const S3 = struct { mutex: std.Io.Mutex = .init, client: *std.http.Client, config: Config, + read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .{}, + closed_handles: std.ArrayList(u32) = .empty, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, base: VFSBase, @@ -184,11 +357,19 @@ pub const S3 = struct { inner: std.Io, http_client: *std.http.Client, config: Config, + opts: InitOpts, ) !S3 { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + return .{ .allocator = allocator, .base = .init(inner), .client = http_client, + .read_pool = read_pool, .config = .{ .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, @@ -241,10 +422,13 @@ pub const S3 = struct { .secret_key = secret_key, .endpoint_url = endpoint, .region = region, - }); + }, .{}); } pub fn deinit(self: *S3) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -350,17 +534,17 @@ pub const S3 = struct { const handle = self.getFileHandle(o.file); const total = self.performRead(handle, o.data, handle.pos) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; }; if (total == 0) { - return .{ .file_read_streaming = std.Io.File.ReadStreamingError.EndOfStream }; + return .{ .file_read_streaming = error.EndOfStream }; } handle.pos += @intCast(total); return .{ .file_read_streaming = total }; }, - .file_write_streaming, .device_io_control => { + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { return self.base.inner.vtable.operate(self.base.inner.userdata, operation); }, } @@ -396,7 +580,11 @@ pub const S3 = struct { fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; return .{ .inode = 0, @@ -413,9 +601,13 @@ pub const S3 = struct { fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; var path_buffer: [8 * 1024]u8 = undefined; const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; @@ -531,7 +723,7 @@ pub const S3 = struct { const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.Reader.Error.Unexpected; + return std.Io.File.ReadPositionalError.Unexpected; }; } @@ -575,7 +767,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ year_day.year, - @intFromEnum(month_day.month), + @backingInt(month_day.month), month_day.day_index + 1, day_seconds.getHoursIntoDay(), day_seconds.getMinutesIntoHour(), @@ -599,9 +791,7 @@ pub const S3 = struct { return try std.fmt.bufPrint(buf, "{s}/{s}/{s}", .{ endpoint, bucket, key }); } - fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { - const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); - + fn listObjectsBody(self: *S3, endpoint: []const u8, bucket: []const u8, key_prefix: []const u8) ![]u8 { var query_buf: [4096]u8 = undefined; var query_writer = std.Io.Writer.fixed(&query_buf); @@ -612,7 +802,9 @@ pub const S3 = struct { if (key_prefix.len > 0) { try query_writer.writeAll("&prefix="); try std.Uri.Component.percentEncode(&query_writer, key_prefix, s3EncodeIsValid); - try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + } } const endpoint_uri = try std.Uri.parse(endpoint); @@ -665,7 +857,16 @@ pub const S3 = struct { return error.RequestFailed; } - const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length orelse 1024 * 1024); + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } + + fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { + const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(endpoint, bucket, key_prefix); defer self.allocator.free(body); return try self.parseListObjectsResponse(body, key_prefix); @@ -679,7 +880,7 @@ pub const S3 = struct { } fn parseListObjectsResponse(self: *S3, xml: []const u8, prefix: []const u8) ![][]const u8 { - var results = std.ArrayListUnmanaged([]const u8){}; + var results: std.ArrayListUnmanaged([]const u8) = .empty; errdefer { for (results.items) |item| self.allocator.free(item); results.deinit(self.allocator); @@ -744,105 +945,53 @@ pub const S3 = struct { var redirect_buffer: [8 * 1024]u8 = undefined; const res = try req.receiveHead(&redirect_buffer); - switch (res.head.status.class()) { - .success => return res.head.content_length.?, - else => { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - return error.ServerError; + return switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + else => blk: { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + break :blk error.ServerError; + }, }, - } + }; } fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - if (offset >= handle.size) return 0; + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; var url_buf: [8 * 1024]u8 = undefined; const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += @as(u64, buf.len); - } - const remaining = handle.size - offset; - const take = @min(remaining, total_bytes); - const end = offset + take - 1; - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .url = url, }; - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - } }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .partial_content and res.head.status != .ok) { - log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); - return error.RequestFailed; + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; } - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } - break :blk null; - }; + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; - const reader = res.reader(&.{}); - - if (content_range) |cr| { - if (cr.start < offset) { - try reader.discardAll(offset - cr.start); - } - } - - return try reader.readSliceShort(data[0]); - } - - const ContentRange = struct { - start: u64, - end: u64, - total: u64, - }; - - fn parseContentRange(value: []const u8) ?ContentRange { - // Format: "bytes start-end/total" - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; + return read_size; } }; diff --git a/vfs/vfs.zig b/vfs/vfs.zig new file mode 100644 index 00000000..fe626ff9 --- /dev/null +++ b/vfs/vfs.zig @@ -0,0 +1,483 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const File = @import("file.zig").File; +pub const GCS = @import("gcs.zig").GCS; +pub const HF = @import("hf.zig").HF; +pub const HTTP = @import("http.zig").HTTP; +pub const S3 = @import("s3.zig").S3; +pub const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs"); + +const CWD_HANDLE: u32 = 0; + +const VFS = @This(); +const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; + +allocator: std.mem.Allocator, +mutex: std.Io.Mutex = .init, + +backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +handles: stdx.SegmentedList(Handle, 128) = .{}, +closed_handles: std.ArrayList(u32) = .empty, + +base: VFSBase, + +pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: @FieldType(@This(), "handles") = .{}; + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + + return .{ + .allocator = allocator, + .handles = handles, + .base = base, + }; +} + +pub fn deinit(self: *VFS) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.backends.deinit(self.allocator); +} + +pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.backends.put(self.allocator, scheme, backend); +} + +pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.backends.remove(scheme); +} + +pub fn io(self: *VFS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; +} + +fn openHandle(self: *VFS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; +} + +fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == CWD_HANDLE) return; + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); +} + +fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { + self.mutex.lockUncancelable(self.base.inner); + const handle = self.handles.at(@intCast(file.handle)); + self.mutex.unlock(self.base.inner); + + return .{ handle, self.getBackend(handle.backend_idx) }; +} + +fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); + return self.handles.at(@intCast(dir.handle)); +} + +fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { + if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; +} + +fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; +} + +fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { + // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it + // is resolved from the scheme's root regardless of `dir`. Without this, + // opening such a path relative to an already-open dir of the same + // backend double-prefixes the dir's path. + if (sub_path) |sp| { + if (std.mem.indexOf(u8, sp, "://") != null) { + const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, uri.scheme, s)) break idx; + } else return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } + } + + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; + + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } + } else { + const handle = self.getDirHandle(dir); + if (handle.backend_idx) |backend_idx| { + return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; + } else { + return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; + } + } +} + +fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; +} + +fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .data = o.data, + } }); + }, + .file_write_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .header = o.header, + .data = o.data, + .splat = o.splat, + } }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } +} + +fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx) }; +} + +fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); +} + +pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + + const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + .flags = file.flags, + }; + return .{ .handle = @intCast(idx), .flags = file.flags }; +} + +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + .flags = file.flags, + }; + return .{ .handle = @intCast(idx), .flags = handle.flags }; +} + +fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; + }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } +} + +fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.Reader.Error.Unexpected; + }; + + const original_dir = reader.dir; + reader.dir = dir_; + defer reader.dir = original_dir; + + return backend.vtable.dirRead(backend.userdata, reader, entries); +} + +fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); + } +} + +fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + } +} + +fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle, const backend = self.getFileHandle(file); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } +} + +fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); +} + +fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + + const original_src_io = reader.io; + const original_src_handle = reader.file; + reader.io = src_backend; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } + + return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); +} + +fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + + const original_src_io = reader.io; + const original_src_handle = reader.file; + reader.io = src_backend; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } + + return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); +} + +fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); +} + +fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); +} + +fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); +} + +fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + + if (self.getScheme(handle.backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); + } +} + +test "copy a local file through the VFS" { + const allocator = std.testing.allocator; + const process_io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source_data: [8192]u8 = @splat('x'); + try tmp.dir.writeFile(process_io, .{ + .sub_path = "source", + .data = &source_data, + }); + + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path_len = try tmp.dir.realPath(process_io, &path_buffer); + const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); + defer allocator.free(source_uri); + const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); + defer allocator.free(destination_uri); + + var file_backend: File = .init(allocator, process_io, .{}); + defer file_backend.deinit(); + var vfs: VFS = try .init(allocator, process_io); + defer vfs.deinit(); + try vfs.register("file", file_backend.io()); + + const vfs_io = vfs.io(); + { + var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); + defer source.close(vfs_io); + var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); + defer destination.close(vfs_io); + + var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); + var copy_buffer: [4096]u8 align(4096) = undefined; + var writer = destination.writer(vfs_io, ©_buffer); + try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); + try writer.interface.flush(); + } + + const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); + defer allocator.free(copied); + try std.testing.expectEqualSlices(u8, &source_data, copied); +} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f037d6b2..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,52 +1,145 @@ -load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", + tags = ["manual"], deps = ["@xla//xla:xla_data_proto"], ) upb_c_proto_library( name = "xla_compile_options_upb", + tags = ["manual"], deps = ["@xla//xla/pjrt/proto:compile_options_proto"], ) +upb_c_proto_library( + name = "profiler_options", + tags = ["manual"], + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], +) + cc_library( name = "posix", hdrs = ["posix.h"], + tags = ["manual"], +) + +cc_library( + name = "profiling_traceme_lib", + srcs = ["profiling/traceme_lib.cc"], + hdrs = ["profiling/traceme.h"], + defines = select({ + "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], + "//conditions:default": [], + }), + linkopts = select({ + "//platforms:rocm.enabled": ["-ldl"], + "//conditions:default": [], + }), + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/lib:traceme", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", + "@xla//xla/tsl/profiler/utils:time_utils", + "@xla//xla/tsl/profiler/utils:xplane_utils", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda:cuda_nvtx_headers", + ], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "@amdrocm-profiler-base//:roctx_headers", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "logo", + srcs = ["logo.zig"], + import_name = "zml/logo", + main = "logo.zig", + visibility = ["//visibility:public"], ) zig_library( name = "zml", srcs = [ - "attention.zig", + "Compiler.zig", + "Sharding.zig", + "attention/Options.zig", "attention/attention.zig", - "attention/flashattn.zig", - "attention/paged_attention.zig", "buffer.zig", + "buffer/HostAccessible.zig", "constants.zig", "dtype.zig", "exe.zig", "floats.zig", "io.zig", + "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", + "kernels/moe/moe.zig", + "kernels/moe/tests.zig", + "kernels/moe/weights.zig", + "kernels/paged_attention/options.zig", + "kernels/paged_attention/paged_attention.zig", + "logo.zig", "mem.zig", "meta.zig", "mlirx.zig", - "module.zig", "nn.zig", "ops.zig", + "ops/manual_computation.zig", "pjrtx.zig", "platform.zig", + "profiling/profiler.zig", + "profiling/tracer.zig", + "quantization.zig", "safetensors.zig", "shape.zig", "slice.zig", "tensor.zig", "testing.zig", - "tracer.zig", "zml.zig", + "//zml/kernel", + "//zml/kernels/flash_attention/cute", + "//zml/kernels/flash_attention/metal", + "//zml/kernels/flash_attention/nki", + "//zml/kernels/moe/cute", + "//zml/kernels/moe/flashinfer_cutlass", + "//zml/kernels/moe/fly", + "//zml/kernels/moe/metal", + "//zml/kernels/moe/mosaic_tpu", + "//zml/kernels/moe/stablehlo", + "//zml/kernels/moe/tcl", + "//zml/kernels/moe/triton", + "//zml/kernels/paged_attention/cute", + "//zml/kernels/paged_attention/metal", + "//zml/kernels/paged_attention/mosaic_tpu", + "//zml/kernels/paged_attention/stablehlo", + "//zml/kernels/paged_attention/tcl", + "//zml/kernels/paged_attention/triton", ], + data = select({ + "//platforms:neuron.enabled": [ + "//zml/kernels/flash_attention/nki:attention_kernel", + ], + "//conditions:default": [], + }), main = "zml.zig", + tags = ["manual"], visibility = ["//visibility:public"], zigopts = [ "-lc", @@ -54,40 +147,53 @@ zig_library( ], deps = [ ":posix", + ":profiler_options", + ":profiling_traceme_lib", ":xla_compile_options_upb", ":xla_data_upb", "//bazel", + "//dsl/cuda_tile:builder", + "//dsl/cute:builder", + "//dsl/fly:builder", + "//dsl/metal:builder", + "//dsl/mosaic_tpu:builder", + "//dsl/tcl:builder", + "//dsl/triton:builder", + "//ffi:zig", "//mlir", - "//mlir/dialects", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/nvvm", + "//mlir/dialects/shardy", + "//mlir/dialects/stablehlo", + "//mlir/dialects/ttir", + "//mlir/dialects/vector", "//pjrt", "//platforms", + "//platforms/tpu:ragged_paged", "//stdx", + "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", "//upb", - "//zml/io", + "//vfs", "//zml/tokenizer", "@rules_zig//zig/runfiles", + "@zio", ] + select({ + "//platforms:neuron.enabled": [ + "//platforms/neuron:nki_kernel", + "//platforms/neuron:topology", + ], + "//conditions:default": [], + }) + select({ "//platforms:cuda.enabled": [ - "//platforms/cuda/flashattn", + "//platforms/cuda/flashinfer_cutlass_moe", ], "//conditions:default": [], }), ) -# All ZML Tests - zig_test( name = "test", - data = [ - #"aio/torch/simple.pt", - #"aio/torch/simple_test_4.pickle", - ], - test_runner = ":test_runner", deps = ["//zml"], ) - -filegroup( - name = "test_runner", - srcs = ["test_runner.zig"], - visibility = ["//visibility:public"], -) diff --git a/zml/Compiler.zig b/zml/Compiler.zig new file mode 100644 index 00000000..93eafa8d --- /dev/null +++ b/zml/Compiler.zig @@ -0,0 +1,1096 @@ +const std = @import("std"); + +const c = @import("c"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const upb = @import("upb"); +const zio = @import("zio"); + +const Buffer = @import("buffer.zig").Buffer; +const DataType = @import("dtype.zig").DataType; +const Exe = @import("exe.zig").Exe; +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; +const TensorStore = @import("io.zig").TensorStore; + +const Compiler = @This(); +const log = std.log.scoped(.@"zml/Compiler"); + +allocator: std.mem.Allocator, +io: std.Io, +arena: std.heap.ArenaAllocator, + +options: Options, +mlir_registry: *mlir.DialectRegistry, +mlir_ctx: *mlir.Context, +mlir_pass_manager: *mlir.PassManager, +module: *mlir.Module, +platform: *const Platform, +meshes: []const *const Sharding.Mesh, +partitioner: Sharding.Partitioner, + +mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), + +scopes: stdx.BoundedArray(Scope, 16) = .empty, +manual_computation_depth: usize = 0, +unknown_location: *const mlir.Location, +location: *const mlir.Location, + +channel_id: i64 = 0, +composite_id: i64 = 0, + +threadlocal var _current: ?*Compiler = null; + +var mlir_global_init_mutex: std.Io.Mutex = .init; +var mlir_global_registry: ?*mlir.DialectRegistry = null; + +fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { + mlir_global_init_mutex.lockUncancelable(io); + defer mlir_global_init_mutex.unlock(io); + + if (mlir_global_registry == null) { + mlir.registerPasses("Transforms"); + + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + func.registerDialect(mlir_registry); + stablehlo.registerDialect(mlir_registry); + shardy.registerDialect(mlir_registry); + func.registerExtensions(mlir_registry); + + mlir_global_registry = mlir_registry; + } + + return mlir_global_registry.?; +} + +pub const Options = struct { + /// Incremented once after successful compilation. + progress: std.Progress.Node = .none, + meshes: []const *const Sharding.Mesh = &.{}, + store: ?*const TensorStore = null, + // If null, will be initialized from the target + partitioner: ?Sharding.Partitioner = null, + /// HACK: Extra bytes added to the device memory size reported to XLA. + /// Some buffers (e.g. engram tables) live in host-pinned memory but are declared to XLA as + /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. + /// XLA then counts them against device memory, leaves no budget for temporary buffers and + /// rematerializes aggressively. Callers set this to the per-device size of such buffers to + /// compensate. Remove once XLA properly supports host-pinned inputs. + extra_device_memory_bytes: u64 = 0, + // Debugging options + program_name: []const u8 = "zml", + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, + xla_dump_emitter_re: ?[]const u8 = null, +}; + +/// Errors surfaced while lowering and compiling a ZML program. +pub const Error = std.mem.Allocator.Error || + std.Io.Writer.Error || + mlir.Error || + upb.SerializeError || + pjrtx.Client.CompileError || + error{ MissingDeviceInTile, OpenShardingAtBoundaries }; + +pub const Donation = union(enum) { + implicit: u32, + explicit: u32, +}; + +pub const Scope = struct { + compiler: *Compiler, + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(Tensor.Id, *const mlir.Value), + id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, Donation), + id_to_memory: std.AutoArrayHashMapUnmanaged(Tensor.Id, Memory.Kind), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(compiler: *Compiler, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(compiler.allocator); + return .{ + .compiler = compiler, + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .id_to_memory = .empty, + .arena = arena, + }; + } + + pub fn pop(self: *Scope) void { + const compiler = self.compiler; + std.debug.assert(self == compiler.currentScope()); + var arena = self.arena; + _ = compiler.scopes.pop(); + arena.deinit(); + } + + pub fn registerTensorAsBlockArgument(scope: *Scope, id: Tensor.Id, arg_id: usize) void { + scope.id_to_argument.put(scope.arena.allocator(), id, scope.block.argument(arg_id)) catch @panic("OOM"); + } + + pub fn value(scope: *Scope, x: Tensor) *const mlir.Value { + return scope.id_to_argument.get(x.id) orelse + x._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) Compiler { + var arena = std.heap.ArenaAllocator.init(allocator); + const mlir_registry = mlirRegistry(io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + const module = mlir.Module.init(unknown_location); + module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); + { + for (0.., &mlir_known_types.values) |i, *mlir_type| { + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); + } + } + + var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; + mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); + if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { + mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); + } + const meshes = mesh_list.items; + validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); + + return .{ + .allocator = allocator, + .io = io, + .arena = arena, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .mlir_known_types = mlir_known_types, + .module = module, + .platform = platform, + .options = opts, + .partitioner = opts.partitioner orelse .fromTarget(platform.target), + .meshes = meshes, + .location = unknown_location, + .unknown_location = unknown_location, + }; +} + +pub fn deinit(self: *Compiler) void { + if (_current == self) _current = null; + std.debug.assert(self.scopes.len == 0); + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.arena.deinit(); +} + +pub fn activate(self: *Compiler) void { + std.debug.assert(_current == null); + _current = self; +} + +pub fn deactivate(self: *Compiler) void { + _ = self; + _current = null; +} + +pub fn current() *Compiler { + return currentOrNull().?; +} + +pub fn currentOrNull() ?*Compiler { + return _current; +} + +pub fn currentScope(self: *Compiler) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; +} + +pub fn pushBlock(self: *Compiler, block: *mlir.Block) *Scope { + const scope = Scope.initFromBlock(self, block); + self.scopes.appendAssumeCapacity(scope); + return self.currentScope(); +} + +/// Change `Compiler.location` by appending a frame to the current stack of locations +/// Must be followed by a `defer compiler.popLocation` +pub fn pushLocation(self: *Compiler, location: std.builtin.SourceLocation, name: []const u8) void { + var callee: *const mlir.Location = .fromSrc(self.mlir_ctx, location); + if (name.len > 0) callee = callee.named(self.mlir_ctx, name); + self.location = if (self.location == self.unknown_location) callee else .callSite(callee, self.location); +} + +/// see `zml.Compiler.pushLocation` +pub fn pushLocationFmt(self: *Compiler, location: std.builtin.SourceLocation, comptime fmt: []const u8, args: anytype) void { + const name = std.fmt.allocPrint(self.allocator, fmt, args) catch { + return self.pushLocation(location, ""); + }; + defer self.allocator.free(name); + + return self.pushLocation(location, name); +} + +/// Remove last call frame pushed by `zml.Compiler.pushLocation` +pub fn popLocation(self: *Compiler) void { + switch (self.location.inspect()) { + .named => { + // We only pushed a named, when there was no parent location + self.location = self.unknown_location; + return; + }, + .callsite => |callsite| { + self.location = callsite.caller(); + return; + }, + .unknown, .file_line_col, .fused => @panic("unexpected location metadata"), + } +} + +test pushLocation { + const mlir_registry = mlirRegistry(std.testing.io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + + var compiler: Compiler = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + .arena = undefined, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = undefined, + .mlir_known_types = undefined, + .module = undefined, + .options = .{}, + .platform = undefined, + .partitioner = undefined, + .meshes = undefined, + .location = unknown_location, + .unknown_location = unknown_location, + }; + + compiler.pushLocation(@src(), "loc1"); + const loc1 = compiler.location; + + compiler.pushLocation(@src(), "loc2"); + compiler.popLocation(); + + try std.testing.expectEqual(loc1, compiler.location); + compiler.popLocation(); + + try std.testing.expectEqual(unknown_location, compiler.location); +} + +pub fn nextChannelId(self: *Compiler) i64 { + self.channel_id += 1; + return self.channel_id; +} + +pub fn nextCompositeId(self: *Compiler) i64 { + self.composite_id += 1; + return self.composite_id; +} + +pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { + return self.mlir_known_types.get(dt); +} + +pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { + @setRuntimeSafety(false); + for (0.., &self.mlir_known_types.values) |i, known_type| { + if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); + } + std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); +} + +pub fn abortOOM(self: *Compiler) noreturn { + _ = self; + @panic("OOM"); +} + +pub fn alloc(self: *Compiler, T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch self.abortOOM(); +} + +pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 { + return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); +} + +/// Lookup a mesh by it's name +pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (compiler.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + if (name == .replicated) return compiler.platform.replicated_mesh; + std.debug.panic( + \\Found no meshes named {s}. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); +} + +/// Find a mesh that covers all the given axes +pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { + if (@TypeOf(logical_axes) != []const Shape.Tag) { + return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); + } + + var ok_meshe: ?*const Sharding.Mesh = null; + for (compiler.meshes) |shd| { + var covers_all: bool = true; + for (logical_axes) |input_axis| { + var covers_this: bool = false; + for (shd.logical.axes.slice()) |existing_axis| { + if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { + covers_this = true; + break; + } + } + covers_all = covers_all and covers_this; + } + if (covers_all) { + if (ok_meshe) |first_match| { + std.debug.panic( + \\Found two meshes covering axes: {any}, expected exacty one. + \\- First match: {f} + \\- Second match: {f} + , .{ logical_axes, first_match, shd }); + } + ok_meshe = shd; + } + } + + return ok_meshe orelse std.debug.panic( + \\Found no meshes covering axes: {any}, expected exacty one. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); +} + +pub fn Typed(comptime func_: anytype) type { + return struct { + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + args: std.meta.ArgsTuple(@TypeOf(func_)), + ) Error!Exe { + return Compiler.compile(allocator, io, platform, func_, args, opts); + } + }; +} + +pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), + opts: Options, +) Error!Exe { + return switch (platform.io_impl) { + .threaded => compileInternal(allocator, io, platform, func_, args, opts), + .zio => try zio.blockInPlace(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { + return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); + } + }.call, .{ allocator, io, platform, args, opts }), + }; +} + +pub fn compileInternal( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), + opts: Options, +) Error!Exe { + // TODO: Here we have somewhat of a requirement + // Emitting MLIR requires to have the compiler context available at all times using `Compiler.current()`. + // If in the future, we inject an Io that is not thread-based, we might have some surprises. + // + // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result + // asynchronously using the provided Io. For now, we'll simply make that blocking as it's not a big deal but keep + // in mind we might want to revisit that later. + // + // This is NOT compatible with std.Io.Evented. + // It is compatible with zio, hence the blockInPlace above. + + const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ + .program_name = opts.program_name, + .arg_count = args.len, + }); + + defer allocator.free(span_name); + var span = tracer.Span.start(span_name); + defer span.end(); + + var compiler: Compiler = .init(allocator, io, platform, opts); + defer compiler.deinit(); + + var result = try compiler.emitMlir(func_, args); + defer result.output_info.deinit(compiler.allocator); + defer result.input_info.deinit(compiler.allocator); + + try addPartitionerOperations(&compiler); + + _ = result.func.appendTo(compiler.module.body()); + + // Mesh replication is encoded in shardings within one SPMD executable. + const num_partitions = compiler.meshes[0].numDevices(); + const num_replicas = 1; + const num_devices = num_partitions * num_replicas; + + compiler.module.operation().setAttributeByName( + "mhlo.num_partitions", + .int(compiler.mlir_ctx, .i32, num_partitions), + ); + compiler.module.operation().setAttributeByName( + "mhlo.num_replicas", + .int(compiler.mlir_ctx, .i32, num_replicas), + ); + + compiler.mlir_pass_manager.runOnOp(compiler.module.operation()) catch |err| switch (err) { + error.MlirUnexpected => { + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Failed to canonicalize invalid MLIR"); + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); + return err; + }; + + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); + + const exe = try Exe.init( + allocator, + platform, + loaded_executable, + num_devices, + num_partitions, + // This will get copied into exe + result.input_info.items(.shape), + result.output_info.items(.shape), + result.input_info.items(.aliasing_output), + ); + errdefer exe.deinit(); + + opts.progress.completeOne(); + return exe; +} + +/// Declare all logical meshes used by this program +fn addPartitionerOperations(ctx: *Compiler) !void { + const mlir_ctx = ctx.mlir_ctx; + const module = ctx.module; + + switch (ctx.partitioner) { + .gspmd => { + // GSPMD doesn't use meshes, but explicit partitioning operations. + }, + .shardy => { + for (ctx.meshes) |shd| { + _ = shardy.mesh(mlir_ctx, shd.name, shd.sdyMeshAttr(mlir_ctx), .unknown(mlir_ctx)).appendTo(module.body()); + } + }, + } +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: std.MultiArrayList(TensorInfo), + output_info: std.MultiArrayList(TensorInfo), +}; + +pub const TensorInfo = struct { + id: Tensor.Id, + shape: Shape, + value: *const mlir.Value, + + // Only used for input tensors, stores which output tensor ends up with their buffer + aliasing_output: ?u32 = null, + + pub fn attributes(info: TensorInfo, arena: std.mem.Allocator, compiler: *const Compiler, scope: *const Scope) error{OutOfMemory}!AttributeList { + var attrs: AttributeList = .empty; + + const mlir_ctx = compiler.mlir_ctx; + const mesh_attr = try compiler.tensorShardingAttr(arena, info.shape); + attrs.appendAssumeCapacity(switch (compiler.partitioner) { + .gspmd => .named(mlir_ctx, "mhlo.sharding", mesh_attr), + .shardy => shardy.shardingNamedAttr(mlir_ctx, mesh_attr.isA(shardy.TensorShardingAttr).?), + }); + + const memory = scope.id_to_memory.get(info.id) orelse .device; + if (memory != .device) { + attrs.appendAssumeCapacity(.named( + mlir_ctx, + "mhlo.memory_kind", + .string(mlir_ctx, compiler.platform.memoryKind(memory)), + )); + } + + if (info.aliasing_output) |output_index| { + attrs.appendAssumeCapacity( + .named(mlir_ctx, "tf.aliasing_output", .int(mlir_ctx, .i32, output_index)), + ); + } + + return attrs; + } +}; + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTuple(@TypeOf(func_))) !EmitMlirResult { + const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const fn_scope: *Scope = compiler.pushBlock(block); + defer fn_scope.pop(); + + var input_info = try createBlockArguments(compiler, fn_scope, &args); + errdefer input_info.deinit(compiler.allocator); + + var output_info = blk: { + compiler.activate(); + defer compiler.deactivate(); + + var result = @call(.auto, func_, args); + // Output collection can emit replication constraints, so it is part of tracing. + break :blk try collectOutputInfo(compiler, fn_scope, &result); + }; + errdefer output_info.deinit(compiler.allocator); + + // Finalize in a separate function that doesn't depend on `func` type. + return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); +} + +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { + const CreateBlockArgumentsCb = struct { + compiler: *Compiler, + scope: *Scope, + + current_argument_id: u32 = 0, + infos: std.MultiArrayList(TensorInfo) = .empty, + + fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { + // Declare the program argument as a packed u8 + const og_shape = tensor._shape; + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); + return error.OpenShardingAtBoundaries; + } + const packed_shape = og_shape.packedShape(); + const mlir_ctx = ctx.compiler.mlir_ctx; + const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); + var location: *const mlir.Location = ctx.compiler.unknown_location; + if (ctx.compiler.options.store) |store| { + if (store.id_to_sources.get(tensor.id)) |sources| { + location = location.named(mlir_ctx, sources.tensors[0].name); + } + } + var value = ctx.scope.block.addArgument(mlir_type, location); + + // But immediately create the MLIR value corresponding to the fp4 data. + // That way the user code only sees a proper fp4 tensor. + if (tensor.dtype().bitSizeOf() < 8) { + value = unpack(ctx.compiler, ctx.scope, og_shape, value); + } + const gop = ctx.scope.id_to_argument.getOrPutValue(ctx.scope.arena.allocator(), tensor.id, value) catch @panic("OOM"); + if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); + + // Associate each input argument with their own buffer + ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, .{ .implicit = ctx.current_argument_id }) catch @panic("OOM"); + + defer ctx.current_argument_id += 1; + + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + .shape = og_shape, + .value = value, + }); + } + }; + + var context: CreateBlockArgumentsCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CreateBlockArgumentsCb.cb, &context, v); + + return context.infos; +} + +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { + const CollectOutputInfoCb = struct { + compiler: *Compiler, + scope: *Scope, + + infos: std.MultiArrayList(TensorInfo) = .empty, + + fn cb(ctx: *@This(), tensor: *const Tensor) !void { + // Outputs with no declared sharding default to replication. Constrain the + // returned value so the partitioner produces a replicated output buffer. + const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); + const og_shape = output.shape(); + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); + return error.OpenShardingAtBoundaries; + } + var value = ctx.scope.id_to_argument.get(output.id) orelse + output._value orelse + @panic("no value found for output tensor"); + + if (tensor.dtype().bitSizeOf() < 8) { + value = repack(ctx.compiler, ctx.scope, og_shape, value); + } + + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + // const packed_shape = og_shape.packedShape(); + // TODO: clarify why this og_shape and not packedShape() + .shape = og_shape, + .value = value, + }); + } + }; + + var context: CollectOutputInfoCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CollectOutputInfoCb.cb, &context, v); + return context.infos; +} + +fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.MultiArrayList(TensorInfo), output_info: std.MultiArrayList(TensorInfo)) error{OutOfMemory}!EmitMlirResult { + var arena_state = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const mlir_ctx = compiler.mlir_ctx; + + const fn_return = func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); + _ = fn_return.appendTo(fn_scope.block); + + // Resolve donations + for (0.., output_info.items(.id)) |output_index, output_id| { + if (fn_scope.id_to_donation.get(output_id)) |donation| { + const donated_input = switch (donation) { + // don't emit implicit donation since they modify the way the function is called + .implicit => continue, + .explicit => |input| input, + }; + const aliasing_output: *?u32 = &input_info.items(.aliasing_output)[donated_input]; + if (aliasing_output.*) |previous_aliased_output| { + std.debug.panic("Input {d} buffer {} was reused twice with `reuseBuffer` for output {} and output {}. Expected `reuseBuffer` to be called at most once", .{ donated_input, input_info.items(.shape)[donated_input], previous_aliased_output, output_index }); + } + aliasing_output.* = @intCast(output_index); + } + } + + // Input mesh/memory/aliasing attributes + const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); + for (0.., input_attributes) |i, *input_attrs| { + const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); + input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); + } + + // Output mesh/memory attributes + const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); + for (0.., output_attributes) |i, *output_attrs| { + const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); + output_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); + } + + const input_types = try arena.alloc(*const mlir.Type, input_info.len); + for (input_types, 0..) |*input_type, i| input_type.* = fn_scope.block.argument(i).type_(); + + const output_types = try arena.alloc(*const mlir.Type, output_info.len); + for (output_types, output_info.items(.value)) |*output_type, value| output_type.* = value.type_(); + + const mlir_func = func.func(mlir_ctx, .{ + .name = "main", + .args = input_types, + .results = output_types, + .block = fn_scope.block, + .location = .unknown(mlir_ctx), + .args_attributes = input_attributes, + .results_attributes = output_attributes, + .verify = false, + }); + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; +} + +fn unpack(compiler: *Compiler, scope: *Scope, og_shape: Shape, tensor_value: *const mlir.Value) *const mlir.Value { + const true_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const bit_cast_op = stablehlo.bitcast_convert( + compiler.mlir_ctx, + tensor_value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const reshape_op = stablehlo.reshape( + compiler.mlir_ctx, + bit_cast_op.result(0), + true_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return reshape_op.result(0); +} + +fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mlir.Value) *const mlir.Value { + const packed_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.packedShape()); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const reshape_op = stablehlo.reshape( + compiler.mlir_ctx, + value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const bit_cast_op = stablehlo.bitcast_convert( + compiler.mlir_ctx, + reshape_op.result(0), + packed_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return bit_cast_op.result(0); +} + +fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { + stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); + + for (meshes) |m| { + const num_replicas = m.numReplicas(); + if (num_replicas > 1 and !std.mem.eql(u8, "replicated", m.name)) { + // This seems like a performance footgun, rather that something anyone would want to use. + // It's generally better to have meshes that cover all devices, + // even if later it means that some mesh axis is never used. + // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas + // TODO(#914): decide if we should forbid this altogether + // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target + log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); + } + log.debug("Using {f}", .{m}); + } + + const first = meshes[0]; + const devices = first.numDevices(); + + for (meshes[1..]) |m| { + if (m.numDevices() != devices) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentMesheCardinality; + } + } +} + +pub fn tensorShardingAttr( + compiler: *const Compiler, + allocator: std.mem.Allocator, + shape: Shape, +) error{OutOfMemory}!*const mlir.Attribute { + const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; + return switch (compiler.partitioner) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).attribute(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, compiler.mlir_ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), + }, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { + const use_shardy_partitioner = switch (compiler.partitioner) { + .shardy => true, + .gspmd => false, + }; + + const platform = compiler.platform; + const main_mesh = compiler.meshes[0]; + // Mesh replicas are part of the SPMD device set, not XLA program replicas. + const num_partitions: i32 = @intCast(main_mesh.numDevices()); + const num_replicas = 1; + + const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); + defer compiler.allocator.free(device_assignment); + + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, true); + c.xla_ExecutableBuildOptionsProto_set_use_shardy_partitioner(exec_build_options, use_shardy_partitioner); + + // Tell XLA how much device memory PJRT actually made available. + // Without this, the GPU scheduler falls back to 80% of physical + // memory. Programs whose donated inputs and outputs exceed that + // artificial limit get a zero-byte temporary-memory budget and + // can trigger excessive rematerialization. + var device_memory_size: ?u64 = null; + for (platform.devices) |device| { + const bytes_limit = device.memoryStats().bytes_limit orelse { + device_memory_size = null; + break; + }; + device_memory_size = @min(device_memory_size orelse bytes_limit, bytes_limit); + } + if (device_memory_size) |bytes_limit| { + // HACK: See `Options.extra_device_memory_bytes`. + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit + opts.extra_device_memory_bytes)); + } + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment_proto = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment_proto, num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment_proto, num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices( + device_assignment_proto, + @intCast(num_partitions), + upb_arena, + ); + + for (computation_devices[0..@intCast(num_partitions)], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids( + computation_device.*, + @intCast(device_assignment[@intCast(i)]), + upb_arena, + ); + } + + break :device_assignment_blk device_assignment_proto; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_scaled_dot_with_tile_ir", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enable for both Blackwell+ and Ampere+ + // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); + }, + .rocm => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); + // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enables fusion to produce output for multiple separate output. + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + // Rearranges operations so that communication between GPUs happens in the background while the compute units are busy doing computation. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + }, + .metal => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_ftz", true, upb_arena); + }, + .oneapi => { + // More efficient for the allgather/broadcast implementation of the collective permute. + try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLASLT,CUSTOM_CALL,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + + // Grouped GEMM still needs a oneDNN lowering. + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); + }, + else => {}, + } + + if (opts.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (opts.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (opts.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + if (opts.xla_dump_emitter_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_emitter_re", re, upb_arena); + } + } + + switch (platform.target) { + .rocm, .cuda, .oneapi => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { + try setXlaOverrideFlag(overrides_map, "xla_gpu_per_fusion_autotune_cache_dir", std.mem.span(path), upb_arena); + }, + else => {}, + } + + break :blk options; + }; + + const loaded_executable = try pjrtx.Client.compile( + platform.pjrt_client, + platform.pjrt_api, + compiler.allocator, + compiler.io, + compiler.module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} + +fn handleCompilationError(compiler: *Compiler, io: std.Io, allocator: std.mem.Allocator, opts: Options, err: anyerror, msg: []const u8) void { + const module_op = compiler.module.operation().fmt(.{ .debug_info = true, .debug_info_pretty_form = true }); + const truncated_msg = "..."; + var too_big: bool = true; + + const buffer: []u8 = allocator.alloc(u8, 8192) catch { + log.err("{s} ({}):\n{f}", .{ msg, err, module_op }); + return; + }; + defer allocator.free(buffer); + + var buffer_writer: std.Io.Writer = .fixed(buffer); + const module_mlir: []const u8 = module_mlir: { + module_op.format(&buffer_writer) catch { + @memcpy(buffer[8192 - truncated_msg.len ..], truncated_msg); + too_big = true; + break :module_mlir buffer; + }; + too_big = false; + break :module_mlir buffer_writer.buffered(); + }; + + log.err("{s} ({}):\n{s}", .{ msg, err, module_mlir }); + if (!too_big) return; + + if (opts.xla_dump_to == null) { + log.warn("To see the full .mlir, set `zml.Compiler.Options.xla_dump_to`", .{}); + return; + } + const xla_dir = opts.xla_dump_to.?; + const dir = std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch dir: { + std.Io.Dir.createDirAbsolute(io, xla_dir, .default_dir) catch |e| { + log.err("failed to dump mlir to {s}: {}", .{ xla_dir, e }); + return; + }; + break :dir std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch unreachable; + }; + const filename = std.fmt.allocPrint(allocator, "{s}.mlir", .{opts.program_name}) catch return; + const mlir_file = dir.createFile(io, filename, .{ .truncate = true }) catch |e| { + log.err("failed to dump mlir to {s}/{s}: {}", .{ xla_dir, filename, e }); + return; + }; + + // Reuse the previously allocated buffer + var mlir_file_writer = mlir_file.writer(io, buffer); + module_op.format(&mlir_file_writer.interface) catch |e| { + log.err("Partial mlir dump at: {s}/{s} ({})", .{ xla_dir, filename, e }); + return; + }; + log.warn("Full .mlir at: {s}/{s}", .{ xla_dir, filename }); +} diff --git a/zml/Sharding.zig b/zml/Sharding.zig new file mode 100644 index 00000000..040216bc --- /dev/null +++ b/zml/Sharding.zig @@ -0,0 +1,3290 @@ +//! Explain how to split a buffer across different devices. +const std = @import("std"); +const builtin = @import("builtin"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +const shardy = @import("mlir/dialects/shardy"); +const stdx = @import("stdx"); + +const Platform = @import("platform.zig").Platform; +const PlatformDevice = @import("platform.zig").Device; +const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; +const Target = @import("platform.zig").Target; + +const Sharding = @This(); + +mesh: ?*const Mesh, +partition: Partitioning, + +pub const MAX_MESH_RANK = 4; + +/// Full replication across participating devices. +/// For buffers, each device holds a complete copy. Applied as a graph constraint, +/// this closes every dimension to partitioning. `Shape.init` uses this sharding. +/// Unconstrained graph results use `open` instead. +pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; + +/// Unspecified sharding for graph results. During compilation, the partitioner +/// may infer sharding from surrounding constraints or leave the value replicated. +/// Executable inputs require closed sharding. An executable output with this +/// unspecified sharding defaults to replication. An explicitly declared output +/// sharding containing open dimensions is rejected. +pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; + +pub fn get(sharding: Sharding, ax: usize) PartitionSpec { + return sharding.partition.get(ax); +} + +pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.set(ax, spec); + return res; +} + +pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.insert(ax, spec); + return res; +} + +pub fn orderedRemove(self: Sharding, ax: usize) Sharding { + var res = self; + res.partition = res.partition.orderedRemove(ax); + return res; +} + +pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { + var result: [Shape.MAX_RANK]PartitionSpec = undefined; + for (&result, 0..) |*spec, ax| spec.* = self.get(ax); + return result; +} + +pub fn isFullyReplicated(self: Sharding) bool { + for (self.partition.toArray()) |spec| { + if (spec != .replicated and spec != .out_of_bound) return false; + } + return true; +} + +/// An unspecified mesh is compatible with any sharding. On the same mesh, +/// open axes accept any in-bounds partition, while closed axes must match. +pub fn isCompatible(self: Sharding, other: Sharding) bool { + if (self.mesh == null or other.mesh == null) return true; + if (self.mesh != other.mesh) return false; + for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { + if (lhs == rhs) continue; + if (lhs == .out_of_bound or rhs == .out_of_bound) return false; + if (lhs != .open and rhs != .open) return false; + } + return true; +} + +test isCompatible { + const tp: Mesh = .initTest(.tp, .{.model}); + const dp: Mesh = .initTest(.dp, .{ .data, .model }); + + const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; + const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ + .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, + .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, + .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, + .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, + }; + for (cases) |case| { + errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); + + try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); + try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); + } +} + +/// Formats the declared sharding, not the partitioner's inferred sharding. +/// Without a mesh, prints `{replicated}` for full replication or `{open}` for +/// unspecified graph sharding. +/// +/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: +/// - `data`: split this dimension over the logical mesh axis named `data`. +/// - `replicated`: keep this dimension whole on each shard. +/// - `?`: leave this dimension to partitioner inference during compilation. +/// - `@dp`: the mesh name. +/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. +pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); + const specs = self.partition.toArray(); + var rank = specs.len; + while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; + + try writer.writeByte('{'); + for (specs[0..rank], 0..) |spec, i| { + if (i > 0) try writer.writeByte(','); + switch (spec) { + .replicated => try writer.writeAll("replicated"), + .open => try writer.writeByte('?'), + .out_of_bound => try writer.writeByte('!'), + else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), + } + } + try writer.print("}}@{s}", .{mesh.name}); +} + +/// Describes how one tensor dimension is distributed across a logical mesh. +/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. +/// In a graph constraint, `.open` lets the partitioner choose either behavior. +/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. +pub const PartitionSpec = enum(u4) { + mesh_axis_0 = 0, + mesh_axis_1 = 1, + mesh_axis_2 = 2, + mesh_axis_3 = 3, + mesh_axis_4 = 4, + mesh_axis_5 = 5, + mesh_axis_6 = 6, + mesh_axis_7 = 7, + // an 8D mesh seems already a lot, the max we know about is 3D. + replicated = 8, + + // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. + out_of_bound = 0xa, + open = 15, + + pub fn sharded(mesh_axis: u3) PartitionSpec { + return @fromBackingInt(@intCast(mesh_axis)); + } + + /// Extract the mesh axis along which we are sharded. Null if not sharded. + pub fn meshAxis(self: PartitionSpec) ?u3 { + const ax = @backingInt(self); + return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; + } + + pub fn isSharded(self: PartitionSpec) bool { + return @backingInt(self) < @backingInt(PartitionSpec.replicated); + } + + pub fn isClosed(self: PartitionSpec) bool { + return @backingInt(self) <= @backingInt(PartitionSpec.replicated); + } + + test isClosed { + try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); + try std.testing.expect(!PartitionSpec.open.isClosed()); + + try std.testing.expect(PartitionSpec.replicated.isClosed()); + + try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); + } +}; + +pub const Partitioner = union(enum) { + shardy, + gspmd, + + pub fn fromTarget(target: Target) Partitioner { + return switch (target) { + .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal, .furiosa => .shardy, + }; + } +}; + +fn shapeHasAxisPartition(shape: Shape) bool { + for (0..shape.rank()) |ax| { + if (shape.partition(ax).meshAxis()) |_| return true; + } + return false; +} + +pub const DimSharding = union(enum) { + sharded: struct { + dim: i64, + factor: u32, + }, + replicated, +}; + +pub fn sdyPerValueShardingAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + shapes: []const Shape, + mesh: *const Mesh, +) error{OutOfMemory}!*const shardy.TensorShardingPerValueAttr { + const meshe_attrs = try allocator.alloc(*const shardy.TensorShardingAttr, shapes.len); + for (meshe_attrs, shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + } + return .get(ctx, meshe_attrs); +} + +pub fn sdyManualAxesAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + in_shapes: []const Shape, + out_shapes: []const Shape, + mesh: *const Mesh, +) error{OutOfMemory}!*const shardy.ManualAxesAttr { + var axis_names = std.ArrayList([]const u8).empty; + defer axis_names.deinit(allocator); + + const Collect = struct { + fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { + for (list.items) |existing| { + if (std.mem.eql(u8, existing, axis_name)) return; + } + list.append(allocator_, axis_name) catch unreachable; + } + }; + + for (in_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + for (out_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + + const axes = try allocator.alloc(*const mlir.StringAttribute, axis_names.items.len); + for (axis_names.items, 0..) |axis_name, i| { + axes[i] = mlir.StringAttribute.init(ctx, axis_name); + } + + return .get(ctx, axes); +} + +pub const Partitioning = packed struct { + _0: PartitionSpec, + _1: PartitionSpec, + _2: PartitionSpec, + _3: PartitionSpec, + _4: PartitionSpec, + _5: PartitionSpec, + _6: PartitionSpec, + _7: PartitionSpec, + + pub const out_of_bound: Partitioning = splat(.out_of_bound); + + const MAX_RANK = Shape.MAX_RANK; + const Vec = @Vector(MAX_RANK, u4); + + pub fn init(specs: []const PartitionSpec) Partitioning { + var res: Partitioning = out_of_bound; + for (0.., specs) |i, spec| { + res = res.set(i, spec); + } + return res; + } + + pub fn replicated(rank_: usize) Partitioning { + return repeat(.replicated, rank_); + } + + pub fn open(rank_: usize) Partitioning { + return repeat(.open, rank_); + } + + pub fn splat(spec: PartitionSpec) Partitioning { + const vec: Vec = @splat(@backingInt(spec)); + return @bitCast(vec); + } + + pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { + std.debug.assert(rank_ <= MAX_RANK); + const splatted: Vec = @splat(@backingInt(spec)); + const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); + return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); + } + + pub fn get(p: Partitioning, ax: usize) PartitionSpec { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + return @fromBackingInt(@truncate(pack >> shift)); + } + + pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const mask = @as(u32, 0xf) << shift; + return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); + } + + pub fn rank(p: Partitioning) u8 { + for (0..MAX_RANK) |i_usize| { + const i: u8 = @truncate(i_usize); + if (p.get(i) == .out_of_bound) return i; + } + return MAX_RANK; + } + + /// Inserts a spec, shifting subsequent slots right and discarding the last slot. + pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); + } + + /// Removes a spec, shifting subsequent slots left and filling the last with unknown. + pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); + } + + pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { + var res: [MAX_RANK]PartitionSpec = undefined; + for (&res, 0..) |*spec, ax| spec.* = p.get(ax); + return res; + } + + pub fn hasUniqueAxes(p: Partitioning) bool { + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; + for (0..MAX_RANK) |shape_ax| { + const spec = p.get(shape_ax); + if (spec.meshAxis()) |mesh_axis| { + if (used_mesh_axes.isSet(mesh_axis)) return false; + used_mesh_axes.set(mesh_axis); + } + } + return true; + } + + test hasUniqueAxes { + try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); + try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); + } + + test "packed access and updates" { + try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); + try std.testing.expectEqual(4, @sizeOf(Partitioning)); + inline for (std.meta.tags(PartitionSpec)) |spec| { + const filled = Partitioning.splat(spec); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(spec, filled.get(ax)); + const updated = Partitioning.out_of_bound.set(ax, spec); + for (0..MAX_RANK) |i| { + try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); + } + } + } + for (0..MAX_RANK + 1) |rank_| { + const parts = Partitioning.replicated(rank_); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); + } + } + } + + test "insertion and removal at every slot" { + var parts: Partitioning = .out_of_bound; + for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); + const original = parts.toArray(); + for (0..MAX_RANK) |ax| { + var inserted = original; + std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); + inserted[ax] = .open; + try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); + + var removed = original; + std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); + removed[MAX_RANK - 1] = .out_of_bound; + try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); + try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); + } + try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); + const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); + try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); + try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); + try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); + } + + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + const logical = if (@TypeOf(partitioning) == Logical) + partitioning + else + // capture all the comptime logic in Logical. + comptime Logical.parse(partitioning); + + return logical.applyUseAll(tags, mesh, default_value); + } + + pub const Logical = union(enum) { + /// Replicates every dimension, including when used as a graph constraint. + replicated, + mapping: struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + }, + + const replicated_tag: Shape.Tag = Shape.toTag(.replicated); + const open_tag: Shape.Tag = Shape.toTag(.open); + + pub fn parse(v: anytype) Logical { + const V = @TypeOf(v); + + var src_: Shape.TagsArray = .empty; + var tgt_: Shape.TagsArray = .empty; + + if (comptime stdx.meta.isStruct(V)) { + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); + tgt_.appendAssumeCapacity(@tagName(fv)); + + if (!comptime stdx.meta.isTuple(V)) { + src_.appendAssumeCapacity(field_name); + } + } + return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; + } + + stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); + } + + test "Logical.parse" { + const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); + + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); + } + + test "Logical.replicated" { + const mesh: Mesh = .initTest(.logical, .{.model}); + const tags: []const Shape.Tag = &.{ "a", "b" }; + const logical: Logical = .replicated; + for ([_]PartitionSpec{ .open, .replicated }) |default| { + try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); + } + const unspecified = Logical.parse(.{}); + try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); + } + + pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + const mapping = switch (logical.*) { + .replicated => return .{ .replicated(tags.len), null }, + .mapping => |*mapping_| mapping_, + }; + + var partition_: Partitioning = .repeat(default_value, tags.len); + var missing_src_axis: ?u8 = null; + + for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + missing_src_axis = @intCast(i); + continue; + }; + + const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) + .replicated + else if (mesh_tag == Logical.open_tag) + .open + else spec: { + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }; + partition_ = partition_.set(shape_ax, spec); + } + return .{ partition_, missing_src_axis }; + } + + pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); + + if (missing_shape_axis) |ax| { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); + } + return p; + } + }; +}; + +/// Device as part of a PhysicalMesh. +pub const Device = struct { + /// Unique device identifier in the mesh + id: u32, + + /// Coordinates in the physical mesh + coords: Coords, + + const Coords = [MAX_MESH_RANK]u8; + // Placeholder coords, that would most likely trigger panic if used like that. + // Those are used when creating a mesh, before calling assignCoords + // Private so it's kept as implementation detail + const undefined_coords: Coords = @splat(0xff); + + fn hasValidCoords(device: Device) bool { + return @as(u32, @bitCast(device.coords)) != @as(u32, @bitCast(undefined_coords)); + } +}; + +/// Physical axis tags represent hardware interconnect tiers. +pub const PhysicalAxisTag = enum { + link, // Intra-device / island-local 1D interconnect + link_x, // Torus/Mesh Dim X (TPU, Trainium) + link_y, // Torus/Mesh Dim Y (TPU, Trainium) + link_z, // Torus/Mesh Dim Z (TPU v3/v4/v5) + bus, // PCIe / NUMA (Potential isolation boundary) +}; + +/// Ring geometry style for 1D neighbor-only interconnects. +pub const RingKind = enum { + linear, // Neighbors only, open ends + closed_ring, // Neighbors only, circular (wrap-around) +}; + +/// Mesh geometry style for 2D/3D neighbor-only interconnects. +pub const MeshKind = enum { + grid, // 2D/3D Neighbors only, open ends + torus, // 2D/3D Neighbors only, circular wrap-around +}; + +/// AxisGeometry describes the communication pattern for an axis. +pub const AxisGeometry = union(enum) { + point_to_point: void, // Full all-to-all (NVLink Switch, TPU v5p) + ring: RingKind, // Neighbor-only 1D + mesh: MeshKind, // Neighbor-only 2D/3D + tree: void, // Hierarchical (Standard PCIe) + isolated: void, // No direct communication (AWS Inf2 islands) + + pub fn format(self: AxisGeometry, writer: *std.Io.Writer) !void { + switch (self) { + .point_to_point => try writer.writeAll("P2P"), + .ring => |k| try writer.print("Ring({s})", .{@tagName(k)}), + .mesh => |k| try writer.print("Mesh({s})", .{@tagName(k)}), + .tree => try writer.writeAll("Tree"), + .isolated => try writer.writeAll("Isolated"), + } + } +}; + +/// Represents the hierarchical physical topology of the hardware devices. +pub const PhysicalNode = union(enum) { + /// A branch represents a physical axis + branch: struct { + tag: PhysicalAxisTag, + geometry: AxisGeometry, + children: []PhysicalNode, + }, + /// A leaf represents the actual hardware device + leaf: Device, + + pub fn axis(tag: PhysicalAxisTag, geometry_: AxisGeometry, children: []const PhysicalNode) PhysicalNode { + return .{ + .branch = .{ + .tag = tag, + .geometry = geometry_, + .children = @constCast(children), + }, + }; + } + + pub fn device(device_: PlatformDevice) PhysicalNode { + return .{ + .leaf = .{ + .id = @intCast(device_.id()), + .coords = Device.undefined_coords, + }, + }; + } + + pub fn countDevices(self: PhysicalNode) usize { + return switch (self) { + .leaf => 1, + .branch => |b| { + var sum: usize = 0; + for (b.children) |child| sum += child.countDevices(); + return sum; + }, + }; + } +}; + +/// PhysicalMesh models hardware as a tree of axes. +/// Each branch = one physical axis (link_x/link_y/bus/etc), +/// each leaf = one device. +/// +/// Diagram: +/// link_x +/// ├─ link_y +/// │ └─ device +/// └─ link_y +/// └─ device +/// +/// The DFS axis order defines canonical coordinate ordering. +pub const PhysicalMesh = struct { + /// Explicit topology tree API (canonical form) + pub const Tree = PhysicalNode; + + /// AxisTraversal captures canonical axis order and depth mapping. + pub const AxisTraversal = struct { + pub const Order = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); + pub const DepthByTag = std.EnumArray(PhysicalAxisTag, ?u8); + + /// Order of axes (DFS, first-child descent). + order: Order, + /// Depth (index) of each axis tag in the order. + depth_by_tag: DepthByTag, + + /// Build axis order and depth mapping from a topology tree. + pub fn init(root: PhysicalNode) AxisTraversal { + var order: Order = .empty; + axisOrderNode(root, &order); + + var depth_by_tag: DepthByTag = .initFill(null); + for (order.slice(), 0..) |t, i| { + depth_by_tag.set(t, @intCast(i)); + } + + return .{ + .order = order, + .depth_by_tag = depth_by_tag, + }; + } + + pub fn depth(self: AxisTraversal, tag: PhysicalAxisTag) ?u8 { + return self.depth_by_tag.get(tag); + } + + /// Canonical DFS axis order: + /// + /// Tree: + /// link_x + /// └─ link_y + /// └─ leaf + /// + /// Order = [link_x, link_y] + /// + /// We descend only the first child to keep a stable axis order that + /// depends on structure (axes), not on device count. + fn axisOrderNode(node: PhysicalNode, out: *Order) void { + switch (node) { + .leaf => {}, + .branch => |b| { + out.appendAssumeCapacity(b.tag); + if (b.children.len > 0) axisOrderNode(b.children[0], out); + }, + } + } + }; + + /// AxisInfo exposes the size and geometry for a tag. + pub const AxisInfo = struct { + tag: PhysicalAxisTag, + size: i64, + geometry: AxisGeometry, + }; + + target: Target, + root: PhysicalNode, + axis_traversal: AxisTraversal, + devices_in_canonical_order: []Device, + + /// Build a PhysicalMesh from a tree: + /// - clone the tree into `allocator` + /// - validate geometry invariants + /// - assign coordinates to leaves + /// - compute axis traversal and canonical device order + pub fn fromTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { + const cloned = try cloneNode(allocator, root); + errdefer freeNode(allocator, cloned); + + return try fromOwnedTree(allocator, target, cloned); + } + + fn fromOwnedTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { + try validateGeometry(root); + + var owned_root = root; + var path: Device.Coords = @splat(0); + assignCoords(&owned_root, &path, 0); + + var mesh: PhysicalMesh = .{ + .target = target, + .root = owned_root, + .axis_traversal = .init(owned_root), + .devices_in_canonical_order = undefined, + }; + try mesh.populateDevicesInCanonicalOrder(allocator); + for (mesh.devices_in_canonical_order) |device| { + std.debug.assert(device.hasValidCoords()); + } + return mesh; + } + + pub fn torus1d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + const nodes = try allocator.alloc(PhysicalNode, devices.len); + for (nodes, devices) |*node, device| { + node.* = .device(device); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, nodes)); + } + + pub fn torus2d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + // Find the best split: 4 -> 2x2, 6 -> 2x3, 8 -> 2x4 + var link_x: usize = 1; + var link_y: usize = devices.len; + if (devices.len < 2) return error.NotEnoughDevices; + + for (2..devices.len) |i| { + if (@mod(devices.len, i) != 0) continue; + const j = @divExact(devices.len, i); + if (i <= j) { + link_x = i; + link_y = j; + } else break; + } + + // This only happen if number of devices is 1 or a prime number + if (link_x == 1) return error.NotEnoughDevices; + + const x_axes = try allocator.alloc(PhysicalNode, link_x); + var device_queue = devices; + for (x_axes) |*x_ax| { + const y_devices = try allocator.alloc(PhysicalNode, link_y); + for (y_devices) |*y_dev| { + y_dev.* = .device(device_queue[0]); + device_queue = device_queue[1..]; + } + + x_ax.* = .axis(.link_y, .{ .mesh = .torus }, y_devices); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, x_axes)); + } + + pub fn torus3d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevices; + if (devices.len > 8) @panic("TODO: adapt logic from torus2D"); + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), + .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), + .device(devices[7]), + }), + }), + }); + + return .fromTree(allocator, target, topology); + } + + fn populateDevicesInCanonicalOrder(self: *PhysicalMesh, allocator: std.mem.Allocator) !void { + var devices_: std.ArrayList(Device) = .empty; + errdefer devices_.deinit(allocator); + try devicesNodeInto(allocator, self.root, &devices_); + self.devices_in_canonical_order = try devices_.toOwnedSlice(allocator); + + const Order = struct { + mesh: *const PhysicalMesh, + fn lessThan(ctx: @This(), a: Device, b: Device) bool { + const ia = ctx.mesh.linearIndexFromCoords(a.coords); + const ib = ctx.mesh.linearIndexFromCoords(b.coords); + return ia < ib; + } + }; + + std.mem.sort(Device, self.devices_in_canonical_order, Order{ .mesh = self }, Order.lessThan); + } + + fn cloneNode(allocator: std.mem.Allocator, node: PhysicalNode) !PhysicalNode { + return switch (node) { + .leaf => |d| .{ + .leaf = .{ + .id = d.id, + .coords = d.coords, + }, + }, + .branch => |b| blk: { + const children = try allocator.alloc(PhysicalNode, b.children.len); + errdefer allocator.free(children); + for (b.children, 0..) |child, i| { + children[i] = try cloneNode(allocator, child); + } + break :blk .{ + .branch = .{ + .tag = b.tag, + .geometry = b.geometry, + .children = children, + }, + }; + }, + }; + } + + fn freeNode(allocator: std.mem.Allocator, node: PhysicalNode) void { + switch (node) { + .leaf => {}, + .branch => |b| { + for (b.children) |child| freeNode(allocator, child); + allocator.free(b.children); + }, + } + } + + pub fn deinit(self: *PhysicalMesh, allocator: std.mem.Allocator) void { + allocator.free(self.devices_in_canonical_order); + freeNode(allocator, self.root); + self.* = undefined; + } + + fn validateGeometry(node: PhysicalNode) !void { + switch (node) { + .leaf => return, + .branch => |b| { + if (b.children.len == 0) return error.InvalidPhysicalMesh; + + switch (b.geometry) { + .mesh => { + var has_leaf = false; + var has_branch = false; + var expected_tag: ?PhysicalAxisTag = null; + + for (b.children) |child| { + switch (child) { + .leaf => has_leaf = true, + .branch => |cb| { + has_branch = true; + if (expected_tag == null) { + expected_tag = cb.tag; + } else if (cb.tag != expected_tag.?) { + return error.IncompatibleGeometry; + } + }, + } + } + + if (has_leaf and has_branch) return error.IncompatibleGeometry; + }, + .ring => { + if (b.children.len < 2) return error.IncompatibleGeometry; + }, + .point_to_point, .tree, .isolated => {}, + } + + for (b.children) |child| try validateGeometry(child); + }, + } + } + + /// Assign coords to each leaf using DFS path indices. + /// + /// Example (2x2): + /// link_x + /// └─ link_y + /// + /// coords: + /// (0,0) (0,1) + /// (1,0) (1,1) + fn assignCoords(node: *PhysicalNode, path: *Device.Coords, depth: usize) void { + switch (node.*) { + .leaf => |*d| { + // Coords can already been set by caller + if (d.hasValidCoords()) return; + + d.coords = path.*; + }, + .branch => |*b| { + for (b.children, 0..) |*child, i| { + path[depth] = @intCast(i); + assignCoords(child, path, depth + 1); + } + }, + } + } + + pub fn countDevices(self: PhysicalMesh) usize { + return self.root.countDevices(); + } + + fn devicesNodeInto(allocator: std.mem.Allocator, node: PhysicalNode, out: *std.ArrayList(Device)) !void { + switch (node) { + .leaf => |d| try out.append(allocator, d), + .branch => |b| { + for (b.children) |child| { + try devicesNodeInto(allocator, child, out); + } + }, + } + } + + /// Canonical DFS linearization for coordinate consensus. + /// + /// sizes: S0..Sk, coords: C0..Ck + /// linear = (((C0 * S1) + C1) * S2 + C2) ... + pub fn linearIndexFromCoords(self: PhysicalMesh, coords: Device.Coords) usize { + const order = self.axisOrder(); + var idx: usize = 0; + for (order.slice()) |tag| { + const depth = self.axis_traversal.depth(tag); + if (depth) |d| { + const coord = coords[@intCast(d)]; + const size = self.axis(tag); + idx = idx * @as(usize, @intCast(size)) + coord; + } + } + return idx; + } + + pub fn axisInfo(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisInfo { + return axisInfoNode(self.root, tag); + } + + fn axisInfoNode(node: PhysicalNode, tag: PhysicalAxisTag) ?AxisInfo { + return switch (node) { + .leaf => null, + .branch => |b| { + if (b.tag == tag) { + return .{ + .tag = b.tag, + .size = @intCast(b.children.len), + .geometry = b.geometry, + }; + } + + for (b.children) |child| { + if (axisInfoNode(child, tag)) |info| return info; + } + + return null; + }, + }; + } + + pub fn hasAxis(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + return self.axisInfo(tag) != null; + } + + pub fn axis(self: PhysicalMesh, tag: PhysicalAxisTag) i64 { + if (self.axisInfo(tag)) |info| return info.size; + + return 1; + } + + pub fn axisOrder(self: PhysicalMesh) AxisTraversal.Order { + return self.axis_traversal.order; + } + + pub fn geometry(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisGeometry { + if (self.axisInfo(tag)) |info| return info.geometry; + + return null; + } + + /// Preferred axis ordering by speed: fastest -> slowest. + /// This is used by Strategy.suggest to match intent to bandwidth. + pub fn shardableAxes(self: PhysicalMesh) []const PhysicalAxisTag { + return switch (self.target) { + .tpu => &.{ .link_x, .link_y, .link_z }, + .neuron => &.{ .link, .link_x, .link_y, .link_z }, + .cuda, .rocm => &.{.link}, + .cpu, .metal, .furiosa => &.{.bus}, + .oneapi => &.{ .link, .bus }, + }; + } + + pub fn isShardable(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + for (self.shardableAxes()) |t| { + if (t == tag) return true; + } + + return false; + } + + pub fn format(self: PhysicalMesh, writer: *std.Io.Writer) !void { + try writer.print("PhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); + try formatNode(self.root, writer, "", 0, true); + } + + fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, depth: u8, is_last: bool) !void { + try writer.print("{s}{s}", .{ prefix, if (is_last) "└── " else "├── " }); + switch (node) { + .leaf => |device| { + try writer.print("Device(id={d}, coords={any})\n", .{ device.id, device.coords[0..depth] }); + }, + .branch => |b| { + try writer.print("[{s}] 0..{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); + + var buf: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, if (is_last) " " else "│ " }) catch unreachable; + + for (b.children, 0..) |child, i| { + const child_is_last = (i == b.children.len - 1); + try formatNode(child, writer, new_prefix, depth +| 1, child_is_last); + } + }, + } + } + + /// CoordsTopology builds a mesh from PJRT `coords` attributes. + const CoordsTopology = struct { + const CoordLayout = struct { + rank: usize, + axis_sizes: [MAX_MESH_RANK]usize, + }; + + const IndexedCoord = struct { + placement: Device, + linear: usize, + }; + + fn parseDevice(device: PlatformDevice) !Device { + const api = device.platform.pjrt_api; + const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; + const coords: Device.Coords = coords: switch (coords_attr) { + .int64list => |values| { + var coords: Device.Coords = @splat(0); + if (values.len == 0 or values.len > MAX_MESH_RANK) return error.InvalidDeviceCoords; + for (coords[0..values.len], values) |*coord, value| { + if (value < 0) return error.InvalidDeviceCoords; + coord.* = @intCast(value); + } + break :coords coords; + }, + else => return error.InvalidDeviceCoords, + }; + + return .{ .id = device.id(), .coords = coords }; + } + + pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]Device { + const placements = try allocator.alloc(Device, platform_devices.len); + for (platform_devices, 0..) |device, i| { + placements[i] = try parseDevice(device); + } + return placements; + } + + pub fn layout(placements: []const Device, max_rank: usize) !CoordLayout { + if (placements.len == 0) return error.InvalidPhysicalMesh; + + const rank = placements[0].coords.len; + if (rank == 0) return error.InvalidDeviceCoords; + if (rank > max_rank) return error.UnsupportedDeviceCoordsRank; + + for (placements[1..]) |coord_placement| { + if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; + } + + var axis_sizes: [MAX_MESH_RANK]usize = @splat(1); + for (0..rank) |ax_i| { + var max_coord: usize = 0; + for (placements) |coord_placement| { + max_coord = @max(max_coord, coord_placement.coords[ax_i]); + } + axis_sizes[ax_i] = max_coord + 1; + } + + return .{ + .rank = rank, + .axis_sizes = axis_sizes, + }; + } + + fn linearIndex(coords: Device.Coords, rank: usize, axis_sizes: []const usize) usize { + var linear: usize = 0; + var stride: usize = 1; + var ax = rank; + while (ax > 0) { + ax -= 1; + linear += coords[ax] * stride; + stride *= axis_sizes[ax]; + } + return linear; + } + + pub fn sorted(allocator: std.mem.Allocator, placements: []const Device, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { + const indexed = try allocator.alloc(IndexedCoord, placements.len); + for (placements, indexed) |coord_placement, *out| { + out.* = .{ + .placement = coord_placement, + .linear = linearIndex(coord_placement.coords, rank, axis_sizes), + }; + } + + const SortCtx = struct { + fn lessThan(_: @This(), a: IndexedCoord, b: IndexedCoord) bool { + return a.linear < b.linear; + } + }; + std.mem.sort(IndexedCoord, indexed, SortCtx{}, SortCtx.lessThan); + + for (indexed[1..], 1..) |entry, i| { + if (entry.linear == indexed[i - 1].linear) return error.InvalidDeviceTopology; + } + + return indexed; + } + + pub fn axisStrides(axis_sizes: []const usize) ![MAX_MESH_RANK]usize { + var strides: [MAX_MESH_RANK]usize = @splat(0); + var stride: usize = 1; + + var i = axis_sizes.len; + while (i > 0) { + i -= 1; + strides[i] = stride; + stride = std.math.mul(usize, stride, axis_sizes[i]) catch return error.InvalidDeviceTopology; + } + + return strides; + } + + pub fn buildMeshNode( + allocator: std.mem.Allocator, + indexed: []const IndexedCoord, + axis_sizes: []const usize, + axis_tags: []const PhysicalAxisTag, + axis_strides: []const usize, + depth: usize, + base_offset: usize, + ) !PhysicalNode { + if (depth == axis_sizes.len) { + const device = &indexed[base_offset].placement; + return .{ .leaf = device.* }; + } + + const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator, child); + allocator.free(children); + } + + for (children, 0..) |*child, i| { + child.* = try buildMeshNode( + allocator, + indexed, + axis_sizes, + axis_tags, + axis_strides, + depth + 1, + base_offset + i * axis_strides[depth], + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags[depth], + .geometry = .{ .mesh = .torus }, + .children = children, + }, + }; + } + }; + + pub fn auto(allocator: std.mem.Allocator, target: Target, platform_devices: []const PlatformDevice) !PhysicalMesh { + const root = try switch (target) { + .cpu => cpu(allocator, platform_devices), + .cuda, .rocm => gpu(allocator, platform_devices), + .tpu => tpu(allocator, platform_devices), + .neuron => return neuron(allocator, platform_devices), + .oneapi, .metal, .furiosa => cpu(allocator, platform_devices), + }; + errdefer freeNode(allocator, root); + + // We check a posteriori that the device id is a valid offset into platform.devices. + // This is generally true because platform.devices is sorted by id and ids are 0-N. + // For platforms where this isn't true like Neuron, + // the platform specific builder need to correctly update the ids so that it's true. + const mesh = try fromOwnedTree(allocator, target, root); + for (mesh.devices_in_canonical_order) |pl| { + const device = platform_devices[pl.id]; + std.debug.assert(pl.hasValidCoords()); + std.debug.assert(device.id() == pl.id); + } + + return mesh; + } + + pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + + for (nodes, platform_devices) |*n, d| n.* = .device(d); + + return .{ + .branch = .{ + .tag = .bus, + .geometry = .tree, + .children = nodes, + }, + }; + } + + pub fn gpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, MAX_MESH_RANK); + const indexed = try CoordsTopology.sorted(allocator, placements, layout.rank, layout.axis_sizes[0..layout.rank]); + defer allocator.free(indexed); + + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + for (nodes, indexed, 0..) |*n, entry, i| { + const linear_coord: Device.Coords = .{ @intCast(i), 0, 0, 0 }; + n.* = .{ .leaf = .{ .id = entry.placement.id, .coords = linear_coord } }; + } + + return .{ + .branch = .{ + .tag = .link, + .geometry = .point_to_point, + .children = nodes, + }, + }; + } + + pub fn tpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + if (platform_devices.len == 0) return error.InvalidPhysicalMesh; + + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, 4); + var rank = layout.rank; + var axis_sizes = layout.axis_sizes; + + while (rank > 1 and axis_sizes[rank - 1] == 1) : (rank -= 1) {} + const indexed = try CoordsTopology.sorted(allocator, placements, rank, axis_sizes[0..rank]); + defer allocator.free(indexed); + + var total_devices: usize = 1; + for (axis_sizes[0..rank]) |axis_size| { + total_devices = std.math.mul(usize, total_devices, axis_size) catch return error.InvalidDeviceTopology; + } + if (total_devices != indexed.len) return error.InvalidDeviceTopology; + for (indexed, 0..) |entry, i| { + if (entry.linear != i) return error.InvalidDeviceTopology; + } + + const axis_tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; + const axis_strides = try CoordsTopology.axisStrides(axis_sizes[0..rank]); + + return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); + } + + pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !PhysicalMesh { + if (comptime !platforms.isEnabled(.neuron)) { + return error.UnsupportedPlatform; + } + + const neuron_topology = @import("platforms/neuron/topology"); + + const nc_ids = try allocator.alloc(usize, platform_devices.len); + defer allocator.free(nc_ids); + for (platform_devices, nc_ids) |device, *nc_id| { + nc_id.* = @intCast(device.localHardwareId()); + } + + var topology = try neuron_topology.visibleMeshFromNcIds(allocator, nc_ids); + defer topology.deinit(allocator); + + var axis_tags: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; + var axis_sizes: stdx.BoundedArray(usize, MAX_MESH_RANK) = .empty; + var axis_geometries: stdx.BoundedArray(AxisGeometry, MAX_MESH_RANK) = .empty; + + for (topology.axes()) |mesh_axis| { + axis_tags.appendAssumeCapacity(switch (mesh_axis.fabric) { + .core => .link, + .chip_x => .link_x, + .chip_y => .link_y, + .chip_z => .link_z, + }); + axis_sizes.appendAssumeCapacity(mesh_axis.size); + axis_geometries.appendAssumeCapacity(switch (mesh_axis.geometry) { + .linear_ring => .{ .ring = .linear }, + .closed_ring => .{ .ring = .closed_ring }, + .torus => .{ .mesh = .torus }, + .point_to_point => .point_to_point, + }); + } + + var coords_buf: [MAX_MESH_RANK]usize = @splat(0); + var next_device: usize = 0; + + const Node = struct { + fn build( + allocator_: std.mem.Allocator, + platform_devices_: []const PlatformDevice, + placements: []const neuron_topology.Placement, + axis_tags_: []const PhysicalAxisTag, + axis_sizes_: []const usize, + axis_geometries_: []const AxisGeometry, + depth: usize, + coords_buf_: *[MAX_MESH_RANK]usize, + next_device_: *usize, + ) !PhysicalNode { + if (depth == axis_tags_.len) { + const placement_ = placements[next_device_.*]; + next_device_.* += 1; + + var coords: Device.Coords = @splat(0); + for (coords_buf_[0..axis_tags_.len], 0..) |coord, i| coords[i] = @intCast(coord); + + return .{ .leaf = .{ .id = @intCast(placement_.nc_id), .coords = coords } }; + } + + const children = try allocator_.alloc(PhysicalNode, axis_sizes_[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator_, child); + allocator_.free(children); + } + + for (children, 0..) |*child, i| { + coords_buf_[depth] = i; + child.* = try @This().build( + allocator_, + platform_devices_, + placements, + axis_tags_, + axis_sizes_, + axis_geometries_, + depth + 1, + coords_buf_, + next_device_, + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags_[depth], + .geometry = axis_geometries_[depth], + .children = children, + }, + }; + } + }; + + const root = try Node.build( + allocator, + platform_devices, + topology.placements, + axis_tags.slice(), + axis_sizes.slice(), + axis_geometries.slice(), + 0, + &coords_buf, + &next_device, + ); + const mesh: PhysicalMesh = try fromOwnedTree(allocator, .neuron, root); + for (mesh.devices_in_canonical_order) |*dev| { + // In Node.build we use nc_id as the Device id. + // But now we want to use the id to get the offset into platform.devices. + const dev_nc_id = dev.id; + for (0.., topology.placements) |order, placement_| { + if (placement_.nc_id == dev_nc_id) { + dev.id = @intCast(order); + std.debug.assert(platform_devices[order].localHardwareId() == dev_nc_id); + break; + } + } + } + return mesh; + } +}; + +pub const LogicalAxisIntent = enum { + /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). + /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). + high_bandwidth, + + /// Needs reliable throughput but can handle some latency (e.g. Shuffling / All-to-All). + /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). + balanced, + + /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). + /// Suitable for system bus or network (PCIe/Ethernet). + low_bandwidth, +}; + +/// Semantic axes, their communication intents, and optional device-count constraints. +pub const LogicalMesh = struct { + pub const Axes = stdx.BoundedArray(Shape.Tag, MAX_MESH_RANK); + pub const Intents = stdx.BoundedArray(LogicalAxisIntent, MAX_MESH_RANK); + + axes: Axes, + intents: Intents, + dimensions: [MAX_MESH_RANK]i32, + + /// Create a logical mesh. + /// * names of the logical axes: .{ .a, .b }, + /// * intents: intent for each axis + /// * dimensions: dimension for each axis, -1 infers the size from underlying topology + pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { + if (@TypeOf(names) != []const Shape.Tag) { + const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); + return .init(parsed_names, intents, dimensions); + } + + if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); + + var result: LogicalMesh = .{ .axes = .init(names), .intents = .init(intents), .dimensions = @splat(-1) }; + for (names, dimensions, 0..) |name, dimension, i| { + if (dimension != -1 and dimension <= 0) return error.InvalidLogicalDimension; + if (std.mem.findScalar(Shape.Tag, names[0..i], name)) |_| { + @panic("invalid logical mesh"); + } + + result.dimensions[i] = dimension; + } + return result; + } + + pub fn mesh(axes_: anytype) LogicalMesh { + const T = @TypeOf(axes_); + + var axes: Axes = .empty; + var intents: Intents = .empty; + + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(axes_, field_name); + axes.appendAssumeCapacity(Shape.toTag(field_name)); + intents.appendAssumeCapacity(intentFromValue(value)); + } + + if (axes.len == 0) { + stdx.debug.panic("LogicalMesh must have at least one axis defined", .{}); + } + + return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; + } + + pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { + return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); + } + + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { + const target = std.mem.span(tag); + for (self.axes.slice(), self.intents.slice()) |t, i| { + if (std.mem.eql(u8, std.mem.span(t), target)) return i; + } + + return null; + } + + fn intentFromValue(v: anytype) LogicalAxisIntent { + const V = @TypeOf(v); + if (V == LogicalAxisIntent) return v; + + if (V == @EnumLiteral()) { + return @field(LogicalAxisIntent, @tagName(v)); + } + + stdx.debug.compileError("LogicalMesh intent must be LogicalAxisIntent, got {}", .{V}); + } + + pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { + try writer.writeAll("LogicalMesh("); + for (self.axes.slice(), self.intents.slice(), 0..) |axis, int, i| { + try writer.print(" {s}={s}", .{ axis, @tagName(int) }); + if (self.dimensions[i] != -1) try writer.print("[{d}]", .{self.dimensions[i]}); + } + try writer.writeAll(")"); + } +}; + +pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); + +pub const Mesh = struct { + pub const zml_no_meta_visit: void = {}; + + name: []const u8, + physical: *const PhysicalMesh, + logical: LogicalMesh, + + grid: DeviceGrid, + + /// Compact binding table: logical axis -> physical axes + bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), + + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { + const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } + } + + pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { + return self.bindings.constSlice()[mesh_axis].constSlice(); + } + + pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { + if (@inComptime()) { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); + } + } else { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (logical_axis == tag) return @truncate(i); + } + } + return null; + } + + pub fn init( + name_: []const u8, + physical: *const PhysicalMesh, + logical: LogicalMesh, + strategy: Strategy, + ) !Mesh { + const num_devices = physical.countDevices(); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); + const axis_order = physical.axisOrder().slice(); + if (axis_order.len == 0) return error.InvalidPhysicalMesh; + if (strategy.bindings.len == 0) return error.InvalidStrategy; + + // Create one binding per logical axis + var bindings: @FieldType(Mesh, "bindings") = .empty; + for (logical.axes.slice()) |_| { + bindings.appendAssumeCapacity(.empty); + } + + for (strategy.bindings.slice()) |strat_binding| { + for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { + for (strat_binding.physical.constSlice()) |part| binding_.appendAssumeCapacity(part.axis); + break; + } + } else { + std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); + } + } + + var mesh: Mesh = .{ + .name = name_, + .logical = logical, + .grid = undefined, + .physical = physical, + .bindings = bindings, + }; + mesh.grid = try DeviceGrid.init(&mesh, &strategy); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); + std.debug.assert(@mod(mesh.numDevices(), mesh.numPartitions()) == 0); + return mesh; + } + + pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); + const axes_tags = Shape.parseTags(tagz); + return .{ + .name = @tagName(name_), + .logical = .{ + .axes = .init(axes_tags), + .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + .dimensions = @splat(-1), + }, + .physical = undefined, + .grid = undefined, + .bindings = .empty, + }; + } + + pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { + return mesh.physical.devices_in_canonical_order; + } + + pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { + return self.physical.linearIndexFromCoords(coords); + } + + pub fn numPartitions(self: *const Mesh) u32 { + var partitions: u32 = 1; + for (self.grid.axis_sizes[0..self.grid.rank]) |size| partitions *= size; + return @intCast(partitions); + } + + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) u32 { + const logical_tag = Shape.toTag(logical_axis); + + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); + return self.grid.axis_sizes[logical_ax]; + } + + pub fn numDevices(self: *const Mesh) u32 { + return @intCast(self.physical.devices_in_canonical_order.len); + } + + pub fn numReplicas(self: *const Mesh) u32 { + return @divExact(self.numDevices(), self.numPartitions()); + } + + pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { + return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); + } + + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { + var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; + var tile_count: usize = 1; + for (self.logical.axes.slice(), 0..) |ax, i| { + const size = self.grid.axis_sizes[i]; + tile_count *= size; + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); + } + const replicas = @divExact(self.grid.device_count, tile_count); + if (replicas > 1) { + axes.appendAssumeCapacity(.get(ctx, .{ .name = self.sdyReplicaAxis(), .size = @intCast(replicas) })); + } + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var ids: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (0..self.grid.device_count) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; + var tile_index: usize = 0; + for (0..self.logical.axes.len) |ax| { + tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; + } + // Device IDs here are executable partition indices, not PJRT device IDs. + ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return .get(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); + } + + fn sdyReplicaAxis(self: *const Mesh) []const u8 { + // There are at most MAX_MESH_RANK logical names, so one candidate is free. + inline for (0..MAX_MESH_RANK + 1) |i| { + const name = std.fmt.comptimePrint("__replica_{d}", .{i}); + const taken = for (self.logical.axes.slice()) |ax| { + if (std.mem.eql(u8, std.mem.span(ax), name)) break true; + } else false; + if (!taken) return name; + } + unreachable; + } + + /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute + /// referencing this mesh. Entries in `[...]` follow tensor dimension order. + /// Mesh axis indices in the specs resolve to this mesh's logical axis names. + /// The tensor rank and logical mesh rank are independent. + /// + /// Each dimension spec is encoded as follows: + /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over + /// logical mesh axis `i`. The dimension is closed to further partitioning. + /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is + /// closed, so the partitioner cannot assign a mesh axis to it. + /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. + /// It may acquire available mesh axes or remain replicated. + /// Closed therefore means no additional mesh axes, not necessarily replicated. + /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. + /// + /// Dimension replication (`{}`) applies to one tensor dimension. The optional + /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis + /// must carry identical tensor data across its coordinates and cannot be used + /// to split any dimension, including an open one. + /// + /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, + /// and a shape whose dimensions are ordered `batch`, `hidden`: + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .open } + /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> + /// ``` + /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. + /// The unused `model` axis stays available because an open dimension exists. + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> + /// ``` + /// Both dimensions are closed. Unused logical axes are explicitly replicated, + /// so devices differing only in their `model` coordinate hold the same shard. + /// + /// ``` + /// Partitioning: .{ .batch = .open, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{?}, {}]> + /// ``` + /// Only `batch` may acquire axes, and both `data` and `model` are available. + /// Adding `replicated={"data", "model"}` would prevent either axis from being + /// inferred for `batch`, despite its `{?}` entry. + pub fn sdyShardingAttrForShape( + data: *const Mesh, + parent_allocator: std.mem.Allocator, + ctx: *mlir.Context, + shape: Shape, + ) error{OutOfMemory}!*const shardy.TensorShardingAttr { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); + const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); + var used: [MAX_MESH_RANK]bool = @splat(false); + const has_open_dimension = shape.containsPartitionSpec(.open); + for (0.., dimensions) |i, *d| { + d.* = switch (shape.partition(i)) { + .out_of_bound => @panic("undefined"), + .replicated => .replicated(ctx), + .open => .open(ctx, &.{}), + else => d: { + const ax = shape.partition(i).meshAxis().?; + used[ax] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); + }, + }; + } + + var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; + var tile_count: usize = 1; + for (data.logical.axes.slice(), 0..) |ax, i| { + tile_count *= data.grid.axis_sizes[i]; + // Open dimensions can acquire unused logical axes during inference. + // Only reserve those axes for replication when every dimension is closed. + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); + } + if (tile_count < data.grid.device_count) { + replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); + } + + return .get(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); + } + + pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); + + var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; + var tile_count: usize = 1; + for (0..shape.rank()) |i| { + const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; + tile_shape.appendAssumeCapacity(size); + tile_count *= size; + } + if (tile_count == 1) return .string(ctx, "{replicated}"); + + // All devices at the same logical tile coordinate are replicas, including + // unused logical axes and physical axes that have no logical binding. + const replicas = @divExact(self.grid.device_count, tile_count); + const has_replication = replicas > 1; + if (has_replication) tile_shape.appendAssumeCapacity(@intCast(replicas)); + + var out: std.Io.Writer.Allocating = .init(allocator); + errdefer out.deinit(); + + const ids = try self.gspmdTileAssignment(allocator, shape); + defer allocator.free(ids); + + try out.writer.writeAll("{devices=["); + for (tile_shape.slice(), 0..) |s, i| { + if (i > 0) try out.writer.writeAll(","); + try out.writer.print("{d}", .{s}); + } + try out.writer.writeAll("]"); + + for (ids, 0..) |id, i| { + if (i == 0) { + try out.writer.print("{d}", .{id}); + } else { + try out.writer.print(",{d}", .{id}); + } + } + + if (has_replication) { + try out.writer.writeAll(" last_tile_dim_replicate"); + } + + try out.writer.writeAll("}"); + + return .string(ctx, try out.toOwnedSlice()); + } + + fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { + _ = try Sharding.Placement.init(&shape); + const devices = self.devicesInCanonicalOrder(); + var tile_axes: stdx.BoundedArray(u8, MAX_MESH_RANK) = .empty; + var used_axes: u8 = 0; + var tile_count: usize = 1; + for (0..shape.rank()) |ax| { + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit != 0) continue; + used_axes |= bit; + tile_axes.appendAssumeCapacity(@intCast(logical_axis)); + tile_count *= self.grid.axis_sizes[logical_axis]; + } + const replicas_per_tile = @divExact(devices.len, tile_count); + const replica_indices = try allocator.alloc(usize, tile_count); + defer allocator.free(replica_indices); + @memset(replica_indices, 0); + const ids = try allocator.alloc(u32, devices.len); + + for (0..devices.len) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; + var tile_index: usize = 0; + for (tile_axes.constSlice()) |logical_axis| { + tile_index = tile_index * self.grid.axis_sizes[logical_axis] + + coords[logical_axis]; + } + // HLO shardings refer to logical partitions; executable device assignment + // separately maps those partitions to physical PJRT device IDs. + ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return ids; + } + + pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { + const count = self.physical.devices_in_canonical_order.len; + + var ids = try allocator.alloc(u32, count); + @memset(ids, std.math.maxInt(u32)); + + for (self.physical.devices_in_canonical_order) |d| { + const coords = d.coords; + const idx = self.physical.linearIndexFromCoords(coords); + ids[idx] = d.id; + } + + for (ids) |id| { + if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; + } + + return ids; + } + + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s}, partitions={d}, replicas={d})\n", .{ self.name, self.numPartitions(), self.numReplicas() }); + + try writer.writeAll("┌ Mappings (logical coordinates from physical coordinates):\n"); + const num_axes = self.logical.axes.len; + for (self.logical.axes.slice(), self.logical.intents.slice(), 0..) |tag, intent, logical_axis| { + try writer.print("{s}── {s} ({s}) -> ", .{ if (logical_axis == num_axes - 1) "└" else "├", tag, @tagName(intent) }); + try self.formatAxisMapping(writer, logical_axis); + try writer.writeAll("\n"); + } + try writer.writeAll("\n"); + try writer.print("┌ {f}\n", .{self.physical}); + + try writer.writeAll("┌ Logical coordinates:\n"); + var coords: [MAX_MESH_RANK]usize = @splat(0); + try self.formatLogicalCoordinates(writer, &coords, 0); + } + + /// Reverse engineer the mapping logical -> physical from the device grid. + fn formatAxisMapping(self: Mesh, writer: *std.Io.Writer, logical_axis: usize) !void { + const Part = struct { + tag: PhysicalAxisTag, + physical_size: usize, + physical_stride: usize, + size: usize, + logical_stride: usize, + }; + // A u8 coordinate has at most eight nontrivial mixed-radix digits. + var parts: stdx.BoundedArray(Part, @bitSizeOf(u8)) = .empty; + const logical_size = self.grid.axis_sizes[logical_axis]; + var logical_stride: usize = 1; + while (logical_stride < logical_size) { + const remaining = @divExact(logical_size, logical_stride); + var size = remaining; + const part: Part = found: { + // Prefer the largest matching portion, so whole axes stay readable. + while (size > 1) : (size -= 1) { + if (remaining % size != 0) continue; + for (self.physical.axisOrder().constSlice()) |tag| { + const physical_size: usize = @intCast(self.physical.axis(tag)); + const depth = self.physical.axis_traversal.depth(tag).?; + var physical_stride: usize = 1; + while (physical_stride * size <= physical_size) : (physical_stride += 1) { + if (physical_size % (physical_stride * size) != 0) continue; + for (self.devicesInCanonicalOrder(), 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[i]; + if ((coords[logical_axis] / logical_stride) % size != + (device.coords[depth] / physical_stride) % size) break; + } else { + break :found .{ + .tag = tag, + .physical_size = physical_size, + .physical_stride = physical_stride, + .size = size, + .logical_stride = logical_stride, + }; + } + } + } + } + try writer.writeAll("device-dependent (see logical coordinates)"); + return; + }; + parts.appendAssumeCapacity(part); + logical_stride *= part.size; + } + if (parts.len == 0) { + try writer.writeAll("0 (replicated)"); + return; + } + var i: usize = parts.len; + while (i > 0) { + i -= 1; + const part = parts.constSlice()[i]; + if (i + 1 < parts.len) try writer.writeAll(" + "); + const partial = part.size != part.physical_size; + if (partial) try writer.writeAll("("); + try writer.writeAll(@tagName(part.tag)); + if (part.physical_stride > 1) try writer.print(" / {d}", .{part.physical_stride}); + if (part.physical_stride * part.size < part.physical_size) try writer.print(" % {d}", .{part.size}); + if (partial) try writer.writeAll(")"); + if (part.logical_stride > 1) try writer.print(" * {d}", .{part.logical_stride}); + } + } + + fn formatLogicalCoordinates(self: Mesh, writer: *std.Io.Writer, coords: *[MAX_MESH_RANK]usize, depth: usize) std.Io.Writer.Error!void { + const axes = self.logical.axes.constSlice(); + if (axes.len == 0) return; + const last = axes.len - 1; + if (depth < last) { + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[depth])); + for (0..size) |coord| { + coords[depth] = coord; + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}={d}\n", .{ axes[depth], coord }); + try self.formatLogicalCoordinates(writer, coords, depth + 1); + } + return; + } + + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}=", .{axes[last]}); + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[last])); + for (0..size) |coord| { + if (coord > 0) try writer.writeAll(","); + try writer.print("{d}", .{coord}); + } + try writer.writeAll(" -> devices="); + for (0..size) |coord| { + coords[last] = coord; + if (coord > 0) try writer.writeAll(","); + var count: usize = 0; + for (0..self.grid.device_count) |device_index| { + if (self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) count += 1; + } + // Keep replicas grouped so each entry corresponds to one last-axis coordinate. + if (count != 1) try writer.writeAll("["); + var written: usize = 0; + for (self.devicesInCanonicalOrder(), 0..) |device, device_index| { + if (!self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) continue; + if (written > 0) try writer.writeAll(","); + try writer.print("{d}", .{device.id}); + written += 1; + } + if (count != 1) try writer.writeAll("]"); + } + try writer.writeAll("\n"); + } + + fn matchesLogicalCoordinates(self: Mesh, device_index: usize, coords: []const usize) bool { + const actual: [MAX_MESH_RANK]u8 = self.grid.logical_coords[device_index]; + for (coords, 0..) |expected, logical_axis| { + if (actual[logical_axis] != expected) return false; + } + return true; + } +}; + +/// Resolved logical coordinates in canonical physical-device order. +/// Unused logical axes have coordinate zero. +pub const DeviceGrid = struct { + pub const Coordinates = [MAX_MESH_RANK]u8; + + comptime { + std.debug.assert(Platform.MAX_NUM_DEVICES <= 256); + } + + rank: u8, + device_count: usize, + axis_sizes: [MAX_MESH_RANK]u32, + logical_coords: [Platform.MAX_NUM_DEVICES]Coordinates align(16), + + fn init(mesh_: *const Mesh, strategy: *const Strategy) !DeviceGrid { + for (strategy.folding.constSlice()) |entry| { + for (entry.sources.constSlice()) |source| { + if (!mesh_.physical.hasAxis(source)) return error.InvalidPhysicalAxis; + } + } + const devices = mesh_.devicesInCanonicalOrder(); + if (devices.len > Platform.MAX_NUM_DEVICES) return error.InvalidPhysicalMesh; + var grid: DeviceGrid = .{ + .rank = @intCast(mesh_.logical.axes.len), + .device_count = devices.len, + .axis_sizes = @splat(1), + .logical_coords = @splat(@splat(0)), + }; + for (mesh_.logical.axes.constSlice(), 0..) |logical_tag, logical_axis| { + var used: std.EnumSet(PhysicalAxisTag) = .empty; + for (strategy.bindings.constSlice()) |binding_| { + if (!std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical_tag))) continue; + for (binding_.physical.constSlice()) |part| { + if (part.size) |size| { + const info = mesh_.physical.axisInfo(part.axis) orelse return error.InvalidPhysicalAxis; + const physical_size: u32 = @intCast(info.size); + if (size == 0 or part.stride == 0 or size > physical_size or part.stride > physical_size / size) return error.InvalidPartialBinding; + if (physical_size % (part.stride * size) != 0) return error.InvalidPartialBinding; + try grid.appendAxis(mesh_, logical_axis, part.axis, part.stride, size); + } else { + for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { + if (used.contains(source)) continue; + used.insert(source); + const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; + try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); + } + } + } + } + const dimension = mesh_.logical.dimensions[logical_axis]; + if (dimension != -1) { + if (dimension <= 0) return error.InvalidLogicalDimension; + const size: u32 = @intCast(dimension); + if (grid.axis_sizes[logical_axis] % size != 0) return error.IncompatibleLogicalDimension; + grid.axis_sizes[logical_axis] = size; + // Keep the least-significant portion of the ordered binding; the rest replicates. + for (grid.logical_coords[0..devices.len]) |*row| { + var coords: [MAX_MESH_RANK]u8 = row.*; + coords[logical_axis] = @intCast(coords[logical_axis] % size); + row.* = coords; + } + } + } + // Shared physical axes are valid only when their portions produce a complete + // Cartesian logical grid with the same number of replicas at every coordinate. + var partitions: usize = 1; + for (grid.axis_sizes[0..grid.rank]) |size| { + if (size > devices.len / partitions) return error.OverlappingLogicalAxes; + partitions *= size; + } + if (devices.len % partitions != 0) return error.OverlappingLogicalAxes; + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (grid.logical_coords[0..devices.len]) |row| { + const coords: [MAX_MESH_RANK]u8 = row; + var index: usize = 0; + for (coords[0..grid.rank], grid.axis_sizes[0..grid.rank]) |coord, size| index = index * size + coord; + counts[index] += 1; + } + for (counts[0..partitions]) |count| { + if (count != devices.len / partitions) return error.OverlappingLogicalAxes; + } + return grid; + } + + fn appendAxis(self: *DeviceGrid, mesh_: *const Mesh, logical_axis: usize, tag: PhysicalAxisTag, stride: u32, size: u32) !void { + if (size > Platform.MAX_NUM_DEVICES / self.axis_sizes[logical_axis]) return error.InvalidPartialBinding; + const depth = mesh_.physical.axis_traversal.depth(tag).?; + self.axis_sizes[logical_axis] *= size; + for (mesh_.devicesInCanonicalOrder(), 0..) |device, i| { + var coords: [MAX_MESH_RANK]u8 = self.logical_coords[i]; + coords[logical_axis] = @intCast(@as(u32, coords[logical_axis]) * size + (device.coords[depth] / stride) % size); + self.logical_coords[i] = coords; + } + } +}; + +pub const Strategy = struct { + pub const Fold = struct { + target: PhysicalAxisTag, + sources: PhysicalList, + }; + pub const PhysicalBinding = struct { + axis: PhysicalAxisTag, + stride: u32 = 1, + /// null binds the whole axis (including any explicit fold). + size: ?u32 = null, + }; + pub const Binding = struct { + logical: Shape.Tag, + physical: stdx.BoundedArray(PhysicalBinding, MAX_MESH_RANK), + }; + + pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); + pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); + + bindings: Bindings, + folding: Folding, + + // TODO: remove .init, promote .parseBindings to .init + // It's currently an error to call .init and never calling `addBinding` + pub const init: Strategy = .{ .bindings = .empty, .folding = .empty }; + + pub fn parseBindings(bindings: anytype) Strategy { + const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; + var res: Strategy = .{ .bindings = .empty, .folding = .empty }; + const info = @typeInfo(@TypeOf(bindings)).@"struct"; + if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (info.field_names, info.field_types) |field_name, Field| { + switch (@typeInfo(Field)) { + .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), + .@"struct" => |struct_info| { + stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); + inline for (@field(bindings, field_name)) |axis_tag| { + res.addBinding(field_name, axis_tag); + } + }, + else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), + } + } + return res; + } + + pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { + self.appendBinding(Shape.toTag(logical), .{ .axis = physical }); + } + + /// Append the mixed-radix portion `(coordinate / stride) % size` of a physical axis. + /// `stride * size` must divide the physical axis size. Partial bindings do not expand folds. + /// Portions are concatenated in insertion order, most significant first. + pub fn addPartialBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag, stride: u32, size: u32) void { + self.appendBinding(Shape.toTag(logical), .{ .axis = physical, .stride = stride, .size = size }); + } + + fn appendBinding(self: *Strategy, logical: Shape.Tag, physical: PhysicalBinding) void { + for (self.bindings.slice()) |*binding_| { + if (std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical))) { + binding_.physical.appendAssumeCapacity(physical); + return; + } + } + self.bindings.appendAssumeCapacity(.{ .logical = logical, .physical = .init(&.{physical}) }); + } + + fn foldSources(self: *const Strategy, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + for (self.folding.constSlice()) |*fold| { + if (fold.target == tag) return fold.sources.constSlice(); + } + return null; + } + + /// Explicitly fold axes: `target` is the kept axis, `sources` define order. + /// If `target` is missing from `sources`, it is prepended. + pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) void { + var list: PhysicalList = .empty; + + var has_target = false; + for (sources) |s| { + if (s == target) { + has_target = true; + break; + } + } + + if (!has_target) list.appendAssumeCapacity(target); + for (sources) |s| list.appendAssumeCapacity(s); + + for (self.folding.slice()) |*f| { + if (f.target == target) { + f.sources = list; + return; + } + } + + self.folding.appendAssumeCapacity(.{ .target = target, .sources = list }); + } + + /// suggest builds a Strategy from logical intents. + pub fn suggest(logical: LogicalMesh, physical: *const PhysicalMesh) Strategy { + var strategy: Strategy = .{ .bindings = .empty, .folding = .empty }; + + const base_order = physical.shardableAxes(); + var available: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; + for (base_order) |tag| { + if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); + } + + if (available.len == 0) { + for (physical.axisOrder().slice()) |tag| available.appendAssumeCapacity(tag); + } + + const avail = available.constSlice(); + if (avail.len == 0) return strategy; + var remaining: [MAX_MESH_RANK]u32 = @splat(1); + for (avail, 0..) |tag, i| remaining[i] = @intCast(physical.axis(tag)); + + // Reserve fixed dimensions before automatic axes can consume their capacity. + // Carve fixed portions from the most-significant end, leaving a contiguous + // low-order portion for automatic axes (e.g. model parallelism on one link). + for ([_]bool{ true, false }) |fixed| { + var counters: std.EnumArray(LogicalAxisIntent, u32) = .initFill(0); + inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { + for (logical.axes.constSlice(), logical.intents.constSlice(), 0..) |tag, intent, logical_axis| { + if (intent != target_intent) continue; + const i = counters.get(target_intent); + counters.set(target_intent, @intCast(i + 1)); + const dimension = logical.dimensions[logical_axis]; + if ((dimension != -1) != fixed) continue; + const preferred = switch (target_intent) { + .high_bandwidth => i % avail.len, + .balanced => (avail.len / 2 + i) % avail.len, + .low_bandwidth => avail.len - 1 - (i % avail.len), + }; + if (fixed) { + if (dimension <= 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, 1); + continue; + } + var needed: u32 = @intCast(dimension); + for (0..avail.len) |offset| { + const axis = if (target_intent == .low_bandwidth) + (preferred + avail.len - offset) % avail.len + else + (preferred + offset) % avail.len; + const size = std.math.gcd(needed, remaining[axis]); + if (size == 1) continue; + remaining[axis] = @divExact(remaining[axis], size); + strategy.addPartialBinding(tag, avail[axis], remaining[axis], size); + needed = @divExact(needed, size); + if (needed == 1) break; + } + // Mesh.init reports an incompatible dimension if capacity was insufficient. + if (needed == @as(u32, @intCast(dimension))) strategy.addPartialBinding(tag, avail[preferred], 1, 1); + } else if (remaining[preferred] > 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, remaining[preferred]); + remaining[preferred] = 1; + } + } + } + } + if (strategy.bindings.len == 0) strategy.addPartialBinding(logical.axes.constSlice()[0], avail[0], 1, 1); + + return strategy; + } +}; + +/// For a given shape, compute the shape of the slice each shard will receive. +pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { + if (shape._sharding.mesh != null) { + const pl: Placement = try Sharding.Placement.init(&shape); + var local_shape = pl.localShape(shape); + local_shape._sharding = .replicated; + return local_shape; + } else { + // fully replicated + return shape; + } +} + +/// Owns per-device addressing metadata for a shape. +pub const Placement = struct { + pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); + pub const Slice1d = struct { + start: i64, + size: i64, + }; + + local_dims: stdx.BoundedArray(i64, Shape.MAX_RANK), + mesh_ptr: ?*const Mesh, + byte_steps: [MAX_MESH_RANK]u64, + + pub fn init(shape: *const Shape) error{IncompatibleSharding}!Placement { + const byte_strides = shape.computeByteStrides(); + return initWithByteStrides(shape, byte_strides.constSlice()); + } + + /// Reuse the caller's contiguous byte strides without recomputing them. + pub fn initWithByteStrides(shape: *const Shape, byte_strides: []const i64) error{IncompatibleSharding}!Placement { + std.debug.assert(byte_strides.len == shape.rank()); + if (builtin.mode == .debug) std.debug.assert(shape._sharding.partition.hasUniqueAxes()); + var result: Placement = .{ + .local_dims = .{ .buffer = undefined, .len = shape.rank() }, + .mesh_ptr = shape._sharding.mesh, + .byte_steps = @splat(0), + }; + for (shape.dims(), result.local_dims.slice(), 0..) |dim, *result_local_dim, ax| { + result_local_dim.* = dim; + const mesh_ = result.mesh_ptr orelse continue; + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const count = mesh_.grid.axis_sizes[logical_axis]; + if (@rem(dim, count) != 0) return error.IncompatibleSharding; + const local_dim = @divExact(dim, count); + result_local_dim.* = local_dim; + result.byte_steps[logical_axis] = @intCast(local_dim * byte_strides[ax]); + } + return result; + } + + pub fn mesh(pl: *const Placement) ?*const Mesh { + return pl.mesh_ptr; + } + + /// Reconstruct metadata only for consumers that need a full shard shape. + pub fn localShape(pl: *const Placement, source: Shape) Shape { + std.debug.assert(source.rank() == pl.local_dims.len); + var result = source; + @memcpy(result._dims.slice(), pl.local_dims.constSlice()); + result._sharding.mesh = pl.mesh_ptr; + return result; + } + + pub fn slices(pl: *const Placement, device_index: usize, source: Shape) Slices { + std.debug.assert(source.rank() == pl.local_dims.len); + var res: Slices = .{ .buffer = undefined, .len = pl.local_dims.len }; + if (pl.mesh_ptr == null) { + for (res.slice(), pl.local_dims.constSlice()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + return res; + } + const grid = &pl.mesh_ptr.?.grid; + std.debug.assert(device_index < grid.device_count); + const coords: [MAX_MESH_RANK]u8 = grid.logical_coords[device_index]; + var used_axes: u8 = 0; + for (res.slice(), pl.local_dims.constSlice(), 0..) |*r, size, ax| { + var coordinate: i64 = 0; + if (source.partition(ax).meshAxis()) |logical_axis| { + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit == 0) coordinate = coords[logical_axis]; + used_axes |= bit; + } + r.* = .{ .start = coordinate * size, .size = size }; + } + return res; + } + + /// Validate the contiguous source slice once before iterating over devices. + /// The returned pointer includes the slice's byte offset. + pub fn basePtr(pl: *const Placement, slice: Slice) [*]const u8 { + if (builtin.mode == .debug) { + std.debug.assert(pl.local_dims.len == slice.shape.rank()); + } + return slice.constData().ptr; + } + + /// device_index is a canonical ordinal, never a PJRT device ID. + /// base_ptr must reference validated contiguous data for the global shape. + pub fn shardPtr(pl: *const Placement, device_index: usize, base_ptr: [*]const u8) [*]const u8 { + return base_ptr + pl.byteOffset(device_index); + } + + fn byteOffset(pl: *const Placement, device_index: usize) usize { + const mesh_ = pl.mesh() orelse return 0; + std.debug.assert(device_index < mesh_.grid.device_count); + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[device_index]; + const steps = pl.byte_steps; + // Placement coefficients and grid coordinates describe offsets within the + // validated source slice. Keep arithmetic checks in debug only. + return offset: { + @setRuntimeSafety(builtin.mode == .debug); + var offset: u64 = 0; + inline for (0..MAX_MESH_RANK) |axis| { + offset += @as(u64, coords[axis]) * steps[axis]; + } + break :offset @intCast(offset); + }; + } + + /// Given a Slice, returns the subslice corresponding to a specific device. + /// Depending on the layout, the sub-slice may or may not be contiguous. + pub fn shardSlice(pl: *const Placement, device_index: usize, slice: Slice) Slice { + const shard_ptr = pl.shardPtr(device_index, pl.basePtr(slice)); + const offset_bytes = @intFromPtr(shard_ptr) - @intFromPtr(slice.bytes.ptr); + return .{ + .bytes = slice.bytes, + .mutable = slice.mutable, + .shape = pl.localShape(slice.shape), + .offset_bytes = offset_bytes, + .byte_strides = slice.byte_strides, + }; + } + + pub fn format(pl: *const Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print( + "Placement(local_dims={any}, mesh={*}, byte_steps={any})\n", + .{ pl.local_dims.constSlice(), pl.mesh_ptr, pl.byte_steps }, + ); + } +}; + +const ShardingTest = struct { + pub const ExpectedShard = struct { + // TODO: remove device_id from here, we always have ids from 0 to N + device_id: usize, + slices: []const [2]i64, + }; + + pub const Scenario = struct { + mesh: *const Mesh, + shape: Shape, + + expected_num_partitions: u32, + expected_num_replicas: u32, + expect_error: ?anyerror = null, + expected_sdy: ?[]const u8 = null, + expected_shards: []const ExpectedShard = &.{}, + + pub fn format(scenario: Scenario, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Scenario: partitions={d}, replicas={d}\n", .{ scenario.expected_num_partitions, scenario.expected_num_replicas }); + if (scenario.expect_error) |err| try writer.print("- error {}\n", .{err}); + if (scenario.expected_sdy) |sdy| try writer.print("- shardy: {s}\n", .{sdy}); + if (scenario.expected_shards.len > 0) try writer.print("- shards:\n", .{}); + for (scenario.expected_shards) |shard| { + try writer.print("\t- device {d}: slices={any}\n", .{ shard.device_id, shard.slices }); + } + } + }; + + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) ShardingTest { + return .{ .allocator = allocator }; + } + + /// Creates a 1D-3D PhysicalMesh from simple dimensions. + pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { + const info = @typeInfo(@TypeOf(dims)).@"struct"; + const N = info.field_names.len; + var sizes: [N]usize = undefined; + inline for (info.field_names, sizes[0..]) |field_name, *s| { + s.* = @intCast(@field(dims, field_name)); + } + + const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; + var next_id: u32 = 0; + const root = try self.buildNode(tags[0..N], sizes[0..N], 0, &next_id, geometry); + return try .fromTree(self.allocator, .tpu, root); + } + + fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *u32, geometry: AxisGeometry) !PhysicalNode { + if (depth == tags.len) { + const id = next_id.*; + next_id.* += 1; + return .{ .leaf = .{ .id = id, .coords = Device.undefined_coords } }; + } + const count = sizes[depth]; + const children = try self.allocator.alloc(PhysicalNode, count); + for (children) |*child| { + child.* = try self.buildNode(tags, sizes, depth + 1, next_id, geometry); + } + return .{ .branch = .{ .tag = tags[depth], .geometry = geometry, .children = children } }; + } + + pub fn run(self: ShardingTest, s: Scenario) !void { + const mesh = s.mesh; + errdefer std.log.warn("Failed mesh {f}", .{s}); + + try std.testing.expectEqual(s.expected_num_partitions, mesh.numPartitions()); + try std.testing.expectEqual(s.expected_num_replicas, mesh.numReplicas()); + try std.testing.expectEqual(mesh.grid.device_count, @as(usize, @intCast(mesh.numDevices()))); + + // Verify MLIR String + if (s.expected_sdy) |expected_attr| { + errdefer std.log.warn("Expected shardy annotation failed", .{}); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try actual_attr.attribute().format(&writer.writer); + try std.testing.expectEqualStrings(expected_attr, writer.written()); + } + + // Verify Placement logic / Error + const ordered_devices = mesh.devicesInCanonicalOrder(); + if (s.expect_error) |err| { + errdefer std.log.warn("Expected error failed", .{}); + try std.testing.expect(ordered_devices.len > 0); + try std.testing.expectError(err, Sharding.Placement.init(&s.shape)); + return; + } + + // Verify Shard Slices (Math) + if (s.expected_shards.len > 0) { + const pl: Placement = try Sharding.Placement.init(&s.shape); + errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); + try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); + for (s.expected_shards, ordered_devices, 0..) |expected, device, device_index| { + { + try std.testing.expectEqual(expected.device_id, device.id); + + // Check pl.slices(device) + const actual_slices = pl.slices(device_index, s.shape); + + // Consider rewriting test cases to use Slice1D. This requires passing the axis + const actual_start_len = try self.allocator.alloc([2]i64, actual_slices.len); + defer self.allocator.free(actual_start_len); + for (actual_start_len, actual_slices.slice()) |*s_l, slice| { + s_l.* = .{ slice.start, slice.size }; + } + + try std.testing.expectEqualSlices([2]i64, expected.slices, actual_start_len); + } + { + // Check pl.shape + const expected_sizes = try self.allocator.alloc(i64, expected.slices.len); + defer self.allocator.free(expected_sizes); + for (expected_sizes, expected.slices) |*expected_size, slice| { + expected_size.* = slice[1]; + } + + try std.testing.expectEqualSlices(i64, expected_sizes, pl.local_dims.constSlice()); + } + } + } + } +}; + +test "mesh: unknown partitioning implies replication" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); + + const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); + const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); + + try runner.run(.{ + .expected_num_partitions = 2, + .expected_num_replicas = 2, + .mesh = &mesh, + // No partitioning on shape + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"batch\", \"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 1, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 3, .slices = &.{.{ 0, 8 }} }, + }, + }); +} + +test "suggest splits a one-dimensional topology for fixed data and automatic model axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }); + for ([_]usize{ 2, 4, 8 }) |count| { + const physical = try runner.physical(.{count}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("DP-MP", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(@as(u32, @intCast(count / 2)), mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (0..count) |i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual(@as(u8, @intCast(i / (count / 2))), coords[0]); + try std.testing.expectEqual(@as(u8, @intCast(i % (count / 2))), coords[1]); + } + const shape = Shape.init(.{ .b = 8, .m = 8 }, .f32).withPartitioning(&mesh_, .{ .b = .data, .m = .model }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = @intCast(count), .expected_num_replicas = 1 }); + } +} + +test "suggest reserves fixed dimensions across physical axes before automatic allocation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 3 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .high_bandwidth, .low_bandwidth }, &.{ -1, 6 }); + const mesh_ = try Mesh.init("split", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + + const fixed = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 3, 4 }); + const fixed_mesh = try Mesh.init("fixed", &physical, fixed, .suggest(fixed, &physical)); + try std.testing.expectEqual(12, fixed_mesh.numPartitions()); + const impossible = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 5, -1 }); + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("impossible", &physical, impossible, .suggest(impossible, &physical))); + const replica_logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 1, 1 }); + const replica_mesh = try Mesh.init("replicated", &physical, replica_logical, .suggest(replica_logical, &physical)); + try std.testing.expectEqual(12, replica_mesh.numReplicas()); +} + +test "mesh: suggest strategy realization" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + + const logical: LogicalMesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }); + + const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); + + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), + .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"batch\"}, {\"model\"}], replicated={\"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 + .{ .device_id = 1, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y0, z1 (batch split) + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y1, z0 (replicated y) + .{ .device_id = 3, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y1, z1 + .{ .device_id = 4, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, // x1, y0, z0 (model split) + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + }, + }); +} + +test "mesh: suggest leaves conflicting logical axes replicated" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{4}, .point_to_point); + + const logical: LogicalMesh = .mesh(.{ + .model = .high_bandwidth, + .experts = .high_bandwidth, + }); + + const strategy: Strategy = .suggest(logical, &physical); + + const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) + .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), + .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"model\"}, {\"experts\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, + .{ .device_id = 1, .slices = &.{ .{ 1, 1 }, .{ 0, 4 } } }, + .{ .device_id = 2, .slices = &.{ .{ 2, 1 }, .{ 0, 4 } } }, + .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, + }, + }); + // Swapping tensor dimensions must not change which logical axis owns the devices. + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .experts = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .experts = .experts, .model = .model }), + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 4 }, .{ 0, 1 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 4 }, .{ 1, 1 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 4 }, .{ 2, 1 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 4 }, .{ 3, 1 } } }, + }, + }); +} + +test "mesh: multiple physical axes on one logical dimension (folding)" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + + { + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } + + { + // Swap link_x and link_y order: + // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } +} + +test "mesh: explicit strategy folding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"model\"}], replicated={\"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 4, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 5, .slices = &.{.{ 12, 4 }} }, + .{ .device_id = 6, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 7, .slices = &.{.{ 12, 4 }} }, + }, + }); +} + +test "mesh: open and replicated dimension mix" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }); + + const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); + + const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + }, + }); +} + +test "mesh: full 3D cluster mesh" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); + + const strategy: Strategy = .parseBindings(.{ + .batch = .link_x, + .model = .link_y, + .context = .link_z, + }); + + const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 8, + .expected_num_replicas = 1, + .mesh = &mesh, + // Note: .context and .model are swapped compared to .link_x and .link_z + .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), + .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"batch\"}, {\"context\"}, {\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 4, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + }, + }); +} + +test "mesh: num partitions for logical axis" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ + .model = .high_bandwidth, + .batch = .low_bandwidth, + }); + + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); + + try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); +} + +test "mesh: size-one and unbound logical axes leave all devices replicated" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 1, 6 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("replicas", &physical, .mesh(.{ .data = .balanced, .model = .balanced }), .parseBindings(.{ .data = .link_x })); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .batch = 12 }, .f32).withPartitioning(&mesh_, .{ .batch = .data }), + .expected_num_partitions = 1, + .expected_num_replicas = 6, + }); +} + +test "mesh: logical partitions and replicas can have different sizes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }), + .expected_num_partitions = 4, + .expected_num_replicas = 3, + }); +} + +test "localized shapes do not retain global sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{2}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); + try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); + const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); + const local = try shardedShape(global); + try std.testing.expectEqual(4, local.dim(.batch)); + try std.testing.expectEqual(null, local._sharding.mesh); + try std.testing.expectEqual(8, global.dim(.batch)); + try std.testing.expectEqual(&mesh_, global._sharding.mesh); +} + +test "GSPMD tiles use tensor axis order and logical partition indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); + const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); + try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); +} + +test "DeviceGrid stores aligned logical coordinates in canonical order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(3 - i); + const mesh_ = try Mesh.init("grid", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .link_y, .data = .link_x })); + const expected = [_][4]u8{ .{ 0, 0, 0, 0 }, .{ 1, 0, 0, 0 }, .{ 0, 1, 0, 0 }, .{ 1, 1, 0, 0 } }; + try std.testing.expectEqual(@as(usize, 0), @intFromPtr(&mesh_.grid.logical_coords) % 16); + for (mesh_.grid.logical_coords[0..mesh_.grid.device_count], expected, 0..) |coords, want, i| { + try std.testing.expectEqual(@intFromPtr(&mesh_.grid.logical_coords) + 4 * i, @intFromPtr(&mesh_.grid.logical_coords[i])); + const actual: [4]u8 = coords; + try std.testing.expectEqualSlices(u8, &want, &actual); + } + try std.testing.expectEqual(@as(usize, 4), @sizeOf(DeviceGrid.Coordinates)); + const shape = Shape.init(.{ .m = 16, .n = 24 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .n = .data }); + const placement = try Sharding.Placement.init(&shape); + const offsets = [_]usize{ 0, 768, 48, 816 }; + var data: [16 * 24 * 4 + 16]u8 = undefined; + var slice = Slice.init(shape, data[16..]); + slice.bytes = &data; + slice.offset_bytes = 16; + const base_ptr = placement.basePtr(slice); + try std.testing.expectEqual(@intFromPtr(&data) + 16, @intFromPtr(base_ptr)); + for (offsets, 0..) |offset, i| { + try std.testing.expectEqual(offset, placement.byteOffset(i)); + try std.testing.expectEqual(@intFromPtr(&data) + 16 + offset, @intFromPtr(placement.shardPtr(i, base_ptr))); + } +} + +test "partial bindings share physical axes and combine portions in order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .balanced, .low_bandwidth }, &.{ -1, -1 }); + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 2, 3); + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 2, 2); + strategy.addPartialBinding(.data, .link_y, 1, 2); + const mesh_ = try Mesh.init("partial", &physical, logical, strategy); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(4, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual((device.coords[1] / 2) * 2 + device.coords[0] % 2, coords[0]); + try std.testing.expectEqual((device.coords[0] / 2) * 2 + device.coords[1] % 2, coords[1]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> (link_y / 2) * 2 + (link_x % 2)") != null); + const shape = Shape.init(.{ .m = 12, .d = 8 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .d = .data }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = 24, .expected_num_replicas = 1 }); +} + +test "fixed logical dimensions constrain whole partial and folded bindings" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data, .unused }, &.{ .balanced, .low_bandwidth, .balanced }, &.{ 2, -1, 1 }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + const mesh_ = try Mesh.init("constrained", &physical, logical, strategy); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numPartitionsForLogicalAxis(.unused)); + try std.testing.expectEqual(2, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual(device.coords[0] % 2, mesh_.grid.logical_coords[i][0]); + } + + const fixed = try LogicalMesh.init(.{.model}, &.{.balanced}, &.{2}); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 4); + const partial = try Mesh.init("partial", &physical, fixed, strategy); + try std.testing.expectEqual(2, partial.numPartitions()); + try std.testing.expectEqual(12, partial.numReplicas()); + + strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_y, .link_x }); + const folded = try Mesh.init("folded", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{8}), strategy); + try std.testing.expectEqual(8, folded.numPartitions()); + try std.testing.expectEqual(3, folded.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual((device.coords[1] * 4 + device.coords[0]) % 8, folded.grid.logical_coords[i][0]); + } + for ([_]i32{ 5, 32 }) |dimension| { + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("invalid", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{dimension}), strategy)); + } +} + +test "partial bindings reject invalid portions and correlated logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{8}, .{ .mesh = .torus }); + for ([_][2]u32{ .{ 0, 2 }, .{ 1, 0 }, .{ 3, 2 }, .{ 1, 3 }, .{ 4, 4 }, .{ 1, std.math.maxInt(u32) } }) |portion| { + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_x, portion[0], portion[1]); + try std.testing.expectError(error.InvalidPartialBinding, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + } + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 1, 2); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 1, 2); + // Four claimed partitions fit on eight devices, but only two coordinates occur. + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "DeviceGrid resolves folds once in strategy order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(11 - i); + var strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_z } }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + strategy.folding = .empty; + + try std.testing.expectEqual(4, mesh_.numPartitions()); + try std.testing.expectEqual(3, mesh_.numReplicas()); + const expected = [_]u8{ 0, 2, 0, 2, 0, 2, 1, 3, 1, 3, 1, 3 }; + for (expected, 0..) |coord, i| { + try std.testing.expectEqual(coord, mesh_.grid.logical_coords[i][0]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> link_z * 2 + link_x\n") != null); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model=0,1,2,3 -> devices=[11,9,7],[5,3,1],[10,8,6],[4,2,0]") != null); +} + +test "DeviceGrid validates fold sources and overlapping folded axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_y, &.{ .link_y, .link_z }); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + + strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "Placement owns dimensions and byte steps across copies" { + const Helper = struct { + fn create() !Placement { + const shape = Shape.init(.{ .m = 3, .n = 5 }, .f32); + return Placement.init(&shape); + } + }; + var original = try Helper.create(); + original.byte_steps[0] = 60; + const copied = original; + original.local_dims.buffer[0] = 99; + original.byte_steps[0] = 100; + try std.testing.expectEqualSlices(i64, &.{ 3, 5 }, copied.local_dims.constSlice()); + try std.testing.expectEqualSlices(u64, &.{ 60, 0, 0, 0 }, &copied.byte_steps); + const source = Shape.init(.{ .m = 3, .n = 5 }, .f32); + try std.testing.expect(copied.localShape(source).eqlWithTags(source)); + var data: [3 * 5 * 4]u8 = undefined; + try std.testing.expectEqual(@intFromPtr(&data), @intFromPtr(copied.basePtr(.init(source, &data)))); +} + +test "Placement byte steps and dot product exceed 32 bits" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{Platform.MAX_NUM_DEVICES}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("wide", &physical, .mesh(.{ .model = .balanced }), .parseBindings(.{ .model = .link_x })); + const shape = Shape.init(.{ .m = 2 * Platform.MAX_NUM_DEVICES, .n = 1 << 30 }, .f32).withPartitioning(&mesh_, .{ .m = .model }); + const placement = try Sharding.Placement.init(&shape); + try std.testing.expectEqual(@as(usize, 1) << 33, placement.byteOffset(1)); + const last_device = Platform.MAX_NUM_DEVICES - 1; + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[last_device]; + try std.testing.expectEqual(@as(u8, @intCast(last_device)), coords[0]); + try std.testing.expectEqual(@as(usize, last_device) << 33, placement.byteOffset(last_device)); +} + +test "DeviceGrid rejects overlapping logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .{ .link_x, .link_y }, .data = .{ .link_y, .link_z } }))); +} + +test "Shardy meshes preserve logical order and unused physical replicas" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("logical", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var text: std.Io.Writer.Allocating = .init(allocator); + defer text.deinit(); + try text.writer.print("{f}", .{mesh_.sdyMeshAttr(ctx)}); + try std.testing.expectEqualStrings( + "#sdy.mesh<[\"batch\"=2, \"model\"=2, \"__replica_0\"=2], device_ids=[0, 2, 4, 6, 1, 3, 5, 7]>", + text.written(), + ); +} + +test "Shardy replica axis avoids logical name collisions" { + const mesh_ = Mesh.initTest(.collision, .{ .__replica_0, .__replica_1 }); + try std.testing.expectEqualStrings("__replica_2", mesh_.sdyReplicaAxis()); +} + +test "GSPMD attributes use logical tile sizes and replica groups" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced, .unused = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + + const Case = struct { shape: Shape, expected: []const u8 }; + const cases = [_]Case{ + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }), + .expected = "{devices=[2,1,4]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .unused }), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{}, .f32), + .expected = "{replicated}", + }, + }; + for (cases) |case| { + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, case.shape); + try std.testing.expect(actual.eql(.string(ctx, case.expected))); + } +} + +test "GSPMD attributes preserve folded logical axis order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .batch = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced, .batch = .balanced }), strategy); + const sharded = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model, .batch = .batch }); + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, sharded); + try std.testing.expect(actual.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7}"))); + + const partial = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }); + const partial_attr = try mesh_.gspmdShardingAttrForShape(allocator, ctx, partial); + try std.testing.expect(partial_attr.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}"))); +} + +test "GSPMD executes a reduction across a folded logical axis" { + if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; + + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = try Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = PhysicalMesh.torus3d }, + }); + defer platform.deinit(allocator, io); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try platform.registerMeshWithStrategy("logical", .mesh(.{ .data = .balanced, .model = .balanced }), strategy); + + const Functions = struct { + fn forward(input: zml.Tensor) zml.Tensor { + return input.sum(.m).squeeze(.m).withPartitioning(input.shape()._sharding.mesh.?, .{ .n = .data }); + } + }; + const input = zml.Tensor.init(.{ .m = 8, .n = 4 }, .i32).withPartitioning(mesh_, .{ .m = .model, .n = .data }); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{input}, .{ + .meshes = &.{mesh_}, + .partitioner = .gspmd, + }); + defer exe.deinit(); + var values: [8][4]i32 = undefined; + var expected: [4]i32 = @splat(0); + for (&values, 0..) |*row, m| { + for (row, 0..) |*value, n| { + value.* = @intCast(10 * m + n); + expected[n] += value.*; + } + } + + var input_buffer = try zml.Buffer.fromBytes(io, platform, input.shape(), std.mem.asBytes(&values)); + defer input_buffer.deinit(); + var output = try exe.eval(allocator, io, .{input_buffer}); + defer output.deinit(); + try std.testing.expectEqual(expected, try output.getValue([4]i32, io)); +} diff --git a/zml/attention.zig b/zml/attention.zig deleted file mode 100644 index 1aa8001b..00000000 --- a/zml/attention.zig +++ /dev/null @@ -1,3 +0,0 @@ -pub const attention = @import("attention/attention.zig"); -pub const paged_attention = @import("attention/paged_attention.zig"); -pub const flashattn = @import("attention/flashattn.zig"); diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index ad747f04..edd9b4ca 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,120 +1,172 @@ const std = @import("std"); const zml = @import("../zml.zig"); -const flashattn = @import("flashattn.zig"); +const flash_attention = @import("../kernels/flash_attention/flash_attention.zig"); -const Attention = @This(); +test { + std.testing.refAllDecls(@This()); +} -pub const Backend = enum { - vanilla, - cuda_fa2, - cuda_fa3, +pub const BackendType = enum { + stablehlo, + flash_attention, +}; + +pub const Backend = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Backend, pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => b: { - const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + return if (flash_attention.isAvailable(platform)) + .{ .flash_attention = flash_attention.Backend.auto(platform) } + else + .{ .stablehlo = {} }; + } - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - break :b if (std.mem.eql(u8, cc, "9.0")) - .cuda_fa3 - else - .cuda_fa2; - } - - break :b .vanilla; - }, - else => .vanilla, + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .flash_attention => |b| b.isAvailable(platform), }; } }; -pub const Parameters = union(Backend) { - vanilla: void, - cuda_fa2: flashattn.fa2.Parameters, - cuda_fa3: flashattn.fa3.Parameters, +pub const Parameters = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters, - pub const InitOptions = union(Backend) { - vanilla: void, - cuda_fa2: flashattn.fa2.Parameters.InitOptions, - cuda_fa3: flashattn.fa3.Parameters.InitOptions, + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => .{ .cuda_fa2 = .{} }, - .cuda_fa3 => .{ .cuda_fa3 = .{} }, + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, }; } }; pub fn init(opts: InitOptions) Parameters { return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, }; } }; -pub const Metadata = union(Backend) { - vanilla: void, - cuda_fa2: flashattn.fa2.Metadata, - cuda_fa3: flashattn.fa3.Metadata, +pub const Metadata = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata, - pub const InitOptions = union(Backend) { - vanilla: void, - cuda_fa2: flashattn.fa2.Metadata.InitOptions, - cuda_fa3: flashattn.fa3.Metadata.InitOptions, + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata.InitOptions, - pub fn fromBackend(backend: Backend, seqlen: i64) InitOptions { + pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { - .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen } }, - .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen } }, + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, }; } }; pub fn init(opts: InitOptions) Metadata { return switch (opts) { - .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, - .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, }; } pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { - .vanilla => .{ .vanilla = {} }, - .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform) }, - .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform) }, + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = try v.initBuffer(io, platform) }, }; } pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { switch (self.*) { - .vanilla => {}, - .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), - .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), + .stablehlo => {}, + .flash_attention => |*v| flash_attention.Metadata.deinitBuffer(v), } } }; -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - return switch (parameters) { - .vanilla => b: { - // Generate the attention mask. - const seq_len = k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); +pub const Options = @import("Options.zig"); - // Note: in Pytorch it would be very inefficient to generate the full attn_mask, - // then slice into it, but XLA is able to optimize this correctly. - attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); - break :b attn_output; - }, - .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), - .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), - }; +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + q: Logical, + k: Logical, + v: Logical, + token_index: Logical = .replicated, + /// Omit to replicate backend metadata tensors. + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, + }, + out_specs: Logical, + + /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. + /// Other dimensions, token indices and backend metadata are replicated. + pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { + const heads: Logical = .parse(.{ .h = mesh_axis }); + return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); } + +/// `attention`, causal or not depending on `opts`. +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ + .in_specs = .{ + .q = partitioning.in_specs.q, + .k = partitioning.in_specs.k, + .v = partitioning.in_specs.v, + .token_index = partitioning.in_specs.token_index, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + }, + .out_specs = partitioning.out_specs, + }); +} + +const Attention = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + metadata: Metadata, + parameters: Parameters, + opts: Options, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .stablehlo => b: { + const seq_len = args.k.dim(.k); + const mask: ?zml.Tensor = if (args.opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) + else + null; + break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); + }, + .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), + }; + } +}; diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig deleted file mode 100644 index e77b27bd..00000000 --- a/zml/attention/flashattn.zig +++ /dev/null @@ -1,1442 +0,0 @@ -const std = @import("std"); - -const flashattn = @import("platforms/cuda/flashattn"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const ffi = zml.pjrt.ffi; - -const AttentionOptions = @import("paged_attention.zig").AttentionOptions; - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { - if (comptime platforms.isEnabled(.cuda)) { - try flashattn.load(allocator, io); - } -} - -pub fn register(platform: *const zml.Platform) !void { - if (comptime platforms.isEnabled(.cuda)) { - try fa2.register(platform); - try fa3.register(platform); - try paged_fa2.Decode.register(platform); - try paged_fa2.Prefill.register(platform); - try paged_fa3.Decode.register(platform); - try paged_fa3.Prefill.register(platform); - } -} - -fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { - return switch (dtype) { - .f32 => .f32, - .f16 => .f16, - .bf16 => .bf16, - .i32 => .i32, - .i8 => .i8, - else => unreachable, - }; -} - -fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !flashattn.Tensor { - return .init( - try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), - buffer.shape().dims(), - buffer.shape().withDtype(.u8).computeStrides().constSlice(), - flashattnDataTypeFromZmlDataType(buffer.dtype()), - ); -} - -const Buffer = struct { - shape: zml.Shape, - ptr: *anyopaque, - - fn toFlashattnTensor(buffer: Buffer) flashattn.Tensor { - return .init( - buffer.ptr, - buffer.shape.dims(), - buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), - flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), - ); - } - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print("{f}@{*}", .{ self.shape, self.ptr }); - } -}; - -fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { - const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; - std.debug.assert(pjrt_api_ptr.dtype == .u64); - const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); - - const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; - std.debug.assert(pjrt_client_ptr.dtype == .u64); - const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); - - return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; -} - -fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { - return switch (ffi_dt) { - .bool => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .f32 => .f32, - .f64 => .f64, - .bf16 => .bf16, - .c64 => .c64, - .c128 => .c128, - .f8e5m2 => .f8e5m2, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3b11fnuz => .f8e4m3b11fnuz, - .f8e5m2fnuz => .f8e5m2fnuz, - .f8e4m3fnuz => .f8e4m3fnuz, - else => unreachable, - }; -} - -fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { - return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); -} - -fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { - return .{ - .shape = shapeFromFfiBuffer(ffi_buffer), - .ptr = ffi_buffer.data, - }; -} - -fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { - const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; - return attribute.get(T); -} - -fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { - var shape = buffer.shape; - const layer_stride = shape.computeByteStrides().get(0); - shape = shape.remove(0); - const ptr = @as([*]u8, @ptrCast(buffer.ptr)); - return .{ - .shape = shape, - .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), - }; -} - -pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { - return struct { - pub fn register(platform: *const zml.Platform) !void { - try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); - } - - pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { - return @field(T, @tagName(run_func))(call_frame) catch b: { - break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); - }; - } - }; -} - -pub const fa2 = struct { - const custom_call_name = "fa2_mha_varlen_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = 0, - .num_heads = num_heads, - }; - - flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - null, - null, - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } - - pub const Parameters = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) fa2.Parameters { - _ = opts; // autofix - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - }; - - pub fn init(opts: InitOptions) Metadata { - return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - _ = parameters; // autofix - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q, const k, const v = if (q_.shape().hasTag(.b) != null) b: { - break :b [_]zml.Tensor{ - q_.merge(.{ .tot = .{ .b, .q } }), k_.merge(.{ .tot = .{ .b, .k } }), v_.merge(.{ .tot = .{ .b, .k } }), - }; - } else b: { - break :b [_]zml.Tensor{ q_.rename(.{ .q = .tot }), k_.rename(.{ .k = .tot }), v_.rename(.{ .k = .tot }) }; - }; - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); - - const original_tot = q.dim(.tot); - const num_heads = q_.dim(.h); - const num_heads_k = k_.dim(.h); - const head_size = q_.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; - if (seqlenq_ngroups_swapped) { - q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); - } - - var o = zml.ops.customCall( - custom_call_name, - .{ - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, - }, - .{q.shape()}, - .{ - //.softmax_scale = @as(f32, 1.0), - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .num_heads = @as(i32, @intCast(num_heads)), - }, - .{ - .output_operand_aliases = &.{0}, - .has_side_effect = true, - }, - ); - - if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); - } - - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); - } -}; - -pub const fa3 = struct { - const custom_call_name = "fa3_mha_fwd"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); - const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = 0, - .sm_margin = 0, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &k.toFlashattnTensor(), - &v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - null, - null, - null, - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - - pub const Parameters = struct { - pub const InitOptions = struct {}; - - pub fn init(opts: InitOptions) fa3.Parameters { - _ = opts; // autofix - return .{}; - } - }; - - pub const Metadata = struct { - softmax_lse: zml.Tensor, - softmax_lse_accum: zml.Tensor, - out_accum: zml.Tensor, - scheduler_metadata: zml.Tensor, - - pub const InitOptions = struct { - seqlen: i64, - }; - - pub fn init(opts: InitOptions) Metadata { - return .{ - .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), - .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), - .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), - .scheduler_metadata = .init(.{2}, .i32), - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return .{ - .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), - .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.softmax_lse.deinit(); - self.softmax_lse_accum.deinit(); - self.out_accum.deinit(); - self.scheduler_metadata.deinit(); - } - }; - - pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { - _ = parameters; // autofix - stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); - const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); - // TODO(Corendos): replace with cumsum - const cu_seqlens_k = b: { - const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); - break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); - }; - const max_seqlen_q: i32 = @intCast(q_.dim(.q)); - const max_seqlen_k: i32 = @intCast(k_.dim(.k)); - var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); - const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); - // TODO(Corendos): replace with cumsum - const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); - - var o = zml.ops.customCall( - custom_call_name, - .{ - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - metadata.softmax_lse, - metadata.softmax_lse_accum, - metadata.out_accum, - metadata.scheduler_metadata, - }, - .{q.shape()}, - .{ - .is_causal = true, - .window_size_left = @as(i32, -1), - .window_size_right = @as(i32, -1), - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - }, - .{ - .output_operand_aliases = &.{0}, - .has_side_effect = true, - }, - ); - - return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); - } -}; - -pub const paged_fa2 = struct { - pub const Variant = enum { - decode, - mixed, - }; - - pub const Options = union(Variant) { - decode: DecodeOptions, - mixed: MixedOptions, - - pub fn isPrefill(self: Options) bool { - return self == .mixed; - } - }; - - pub const Parameters = union(Variant) { - decode: DecodeParameters, - mixed: MixedParameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, - .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - .decode => |decode| decode.allocationSize(), - .mixed => |mixed| mixed.allocationSize(), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - .decode => |v| .{ .decode = v.options }, - .mixed => |v| .{ .mixed = v.options }, - }; - } - }; - - pub const DecodeOptions = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - head_dim: usize, - }; - - pub const DecodeParameters = struct { - block_table: zml.Tensor, - cu_seqlens_q: zml.Tensor, - seqused_k: zml.Tensor, - - metadata: DecodeMetadata, - options: DecodeOptions, - - pub fn init(options: DecodeOptions) DecodeParameters { - return .{ - .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), - .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), - .seqused_k = .init(.{options.batch_size}, .i32), - .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), - .options = options, - }; - } - - pub fn allocationSize(self: DecodeParameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.cu_seqlens_q.byteSize(); - allocation_size += self.seqused_k.byteSize(); - allocation_size += self.metadata.allocationSize(); - - return allocation_size; - } - }; - - pub const DecodeMetadata = struct { - out_accum: zml.Tensor, - - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { - return .{ - .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), - }; - } - - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { - self.out_accum.deinit(); - } - - pub fn allocationSize(self: DecodeMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - return allocation_size; - } - }; - - pub const MixedOptions = struct { - batch_size_prefill: usize, - batch_size_decode: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - head_dim: usize, - }; - - pub const MixedParameters = struct { - block_table_prefill: zml.Tensor, - cu_seqlens_q_prefill: zml.Tensor, - seqused_k_prefill: zml.Tensor, - - block_table_decode: zml.Tensor, - cu_seqlens_q_decode: zml.Tensor, - seqused_k_decode: zml.Tensor, - - metadata: MixedMetadata, - options: MixedOptions, - - pub fn init(options: MixedOptions) MixedParameters { - return .{ - .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), - .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), - .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), - .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), - .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), - .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), - .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), - .options = options, - }; - } - - pub fn allocationSize(self: MixedParameters) usize { - var allocation_size: usize = 0; - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_prefill.byteSize(); - allocation_size += self.seqused_k_prefill.byteSize(); - - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_decode.byteSize(); - allocation_size += self.seqused_k_decode.byteSize(); - allocation_size += self.metadata.allocationSize(); - - return allocation_size; - } - }; - - pub const MixedMetadata = struct { - out_accum: zml.Tensor, - host_metadata: zml.Tensor, - - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { - return .{ - .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), - .host_metadata = .init(.{2}, .i32), - }; - } - - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .host_metadata = undefined, - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { - self.out_accum.deinit(); - } - - pub fn allocationSize(self: MixedMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - allocation_size += self.host_metadata.byteSize(); - return allocation_size; - } - }; - - pub const Context = struct { - max_seqlen_k: usize, - decode_offset: ?zml.Tensor = null, - - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; // autofix - _ = num_kv_heads; // autofix - _ = head_dim; // autofix - _ = page_size; // autofix - const max_seqlen_k = switch (parameters) { - inline else => |v| v.options.max_seqlen_k, - }; - const decode_offset = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), - .decode => null, - }; - return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; - } - }; - - pub const Prefill = struct { - pub const custom_call_name = "paged_fa2_prefill"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = 0, - .num_heads = num_heads, - }; - - flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } - }; - - pub const Decode = struct { - pub const custom_call_name = "paged_fa2_decode"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA2MhaVarlenFwdParams = .{ - .max_seqlen_q = 1, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .num_splits = 0, - .num_heads = num_heads, - }; - - flashattn.fa2_mha_varlen_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - &cu_seqlens_k.toFlashattnTensor(), - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - &softmax_lse.toFlashattnTensor(), - null, - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - ¶ms, - stream, - ); - - return null; - } - }; - - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - - const o = switch (parameters) { - .decode => |decode_parameters| b: { - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); - - const original_tot = q.dim(.b); - const num_heads = q.dim(.h); - const num_heads_k = k_cache.dim(.h); - const head_size = q.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; - var q2 = q; - if (seqlenq_ngroups_swapped) { - q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); - } - - var o = zml.ops.customCall( - Decode.custom_call_name, - .{ - q2, - k_cache, - v_cache, - decode_parameters.cu_seqlens_q, - dummy_cu_seqlens_k, - decode_parameters.seqused_k, - decode_parameters.block_table, - softmax_lse, - softmax_lse_accum, - decode_parameters.metadata.out_accum, - layer_index, - }, - .{q2.shape()}, - .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = num_heads, - .window_size_left = opts.sliding_window, - }, - .{ .has_side_effect = false }, - ); - - if (seqlenq_ngroups_swapped) { - o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); - } - - break :b o; - }, - .mixed => |mixed_parameters| b: { - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); - var o = zml.ops.customCall( - Prefill.custom_call_name, - .{ - q, - k_cache, - v_cache, - mixed_parameters.cu_seqlens_q_prefill, - dummy_cu_seqlens_k_prefill, - mixed_parameters.seqused_k_prefill, - mixed_parameters.block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - mixed_parameters.metadata.out_accum, - mixed_parameters.metadata.host_metadata, - layer_index, - }, - .{q.shape()}, - .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = q.dim(.h), - .window_size_left = opts.sliding_window, - }, - .{ .has_side_effect = false }, - ); - - const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); - const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); - - const original_tot = q_decode.dim(.b); - const num_heads = q_decode.dim(.h); - const num_heads_k = k_cache.dim(.h); - const head_size = q_decode.dim(.hd); - const ngroups = @divExact(num_heads, num_heads_k); - const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; - if (seqlenq_ngroups_swapped) { - q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); - } - - var o_decode = zml.ops.customCall( - Decode.custom_call_name, - .{ - q_decode, - k_cache, - v_cache, - mixed_parameters.cu_seqlens_q_decode, - dummy_cu_seqlens_k_decode, - mixed_parameters.seqused_k_decode, - mixed_parameters.block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - mixed_parameters.metadata.out_accum, - layer_index, - }, - .{q_decode.shape()}, - .{ - .is_causal = opts.is_causal, - .max_seqlen_k = context.max_seqlen_k, - .num_heads = q.dim(.h), - .window_size_left = opts.sliding_window, - }, - .{ .has_side_effect = false }, - ); - - if (seqlenq_ngroups_swapped) { - o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); - } - - o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); - break :b o; - }, - }; - - return o; - } -}; - -pub const paged_fa3 = struct { - pub const Variant = enum { - decode, - mixed, - }; - - pub const Options = union(Variant) { - decode: DecodeOptions, - mixed: MixedOptions, - - pub fn isPrefill(self: Options) bool { - return self == .mixed; - } - }; - - pub const Parameters = union(Variant) { - decode: DecodeParameters, - mixed: MixedParameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, - .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - .decode => |decode| decode.allocationSize(), - .mixed => |mixed| mixed.allocationSize(), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - .decode => |v| .{ .decode = v.options }, - .mixed => |v| .{ .mixed = v.options }, - }; - } - }; - - pub const DecodeOptions = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - head_dim: usize, - }; - - pub const DecodeParameters = struct { - block_table: zml.Tensor, - cu_seqlens_q: zml.Tensor, - seqused_k: zml.Tensor, - - metadata: DecodeMetadata, - options: DecodeOptions, - - pub fn init(options: DecodeOptions) DecodeParameters { - return .{ - .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), - .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), - .seqused_k = .init(.{options.batch_size}, .i32), - .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), - .options = options, - }; - } - - pub fn allocationSize(self: DecodeParameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.cu_seqlens_q.byteSize(); - allocation_size += self.seqused_k.byteSize(); - allocation_size += self.metadata.allocationSize(); - - return allocation_size; - } - }; - - pub const DecodeMetadata = struct { - out_accum: zml.Tensor, - - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { - return .{ - .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), - }; - } - - pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { - self.out_accum.deinit(); - } - - pub fn allocationSize(self: DecodeMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - return allocation_size; - } - }; - - pub const MixedOptions = struct { - batch_size_prefill: usize, - batch_size_decode: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - head_dim: usize, - }; - - pub const MixedParameters = struct { - block_table_prefill: zml.Tensor, - cu_seqlens_q_prefill: zml.Tensor, - seqused_k_prefill: zml.Tensor, - - block_table_decode: zml.Tensor, - cu_seqlens_q_decode: zml.Tensor, - seqused_k_decode: zml.Tensor, - - metadata: MixedMetadata, - options: MixedOptions, - - pub fn init(options: MixedOptions) MixedParameters { - return .{ - .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), - .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), - .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), - .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), - .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), - .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), - .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), - .options = options, - }; - } - - pub fn allocationSize(self: MixedParameters) usize { - var allocation_size: usize = 0; - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_prefill.byteSize(); - allocation_size += self.seqused_k_prefill.byteSize(); - - allocation_size += self.block_table_decode.byteSize(); - allocation_size += self.cu_seqlens_q_decode.byteSize(); - allocation_size += self.seqused_k_decode.byteSize(); - allocation_size += self.metadata.allocationSize(); - - return allocation_size; - } - }; - - pub const MixedMetadata = struct { - out_accum: zml.Tensor, - host_metadata: zml.Tensor, - - pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { - return .{ - .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), - .host_metadata = .init(.{2}, .i32), - }; - } - - pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { - return .{ - .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), - .host_metadata = undefined, - }; - } - - pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { - self.out_accum.deinit(); - } - - pub fn allocationSize(self: MixedMetadata) usize { - var allocation_size: usize = 0; - allocation_size += self.out_accum.byteSize(); - allocation_size += self.host_metadata.byteSize(); - return allocation_size; - } - }; - - pub const Context = struct { - max_seqlen_k: usize, - decode_offset: ?zml.Tensor = null, - - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - _ = num_heads; // autofix - _ = num_kv_heads; // autofix - _ = head_dim; // autofix - _ = page_size; // autofix - const max_seqlen_k = switch (parameters) { - inline else => |v| v.options.max_seqlen_k, - }; - const decode_offset = switch (parameters) { - .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), - .decode => null, - }; - return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; - } - }; - - pub const Prefill = struct { - pub const custom_call_name = "paged_fa3_prefill"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = max_seqlen_q, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = 0, - .sm_margin = 0, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - null, - null, - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - }; - - pub const Decode = struct { - pub const custom_call_name = "paged_fa3_decode"; - const Wrapped = Wrapper(@This(), .runInner); - - const register = Wrapped.register; - const run = Wrapped.run; - - pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { - if (call_frame.registeringHook()) return null; - - const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); - const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; - - const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); - const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); - const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); - const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); - const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); - const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); - const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); - const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); - const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); - const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); - const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); - - const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { - const head_dim = q.shape.dim(2); - break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); - }; - const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; - const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; - const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; - const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; - - const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); - const stream = call_frame.api.stream(ctx); - - const params: flashattn.FA3MhaFwdParams = .{ - .max_seqlen_q = 1, - .max_seqlen_k = max_seqlen_k, - .is_causal = is_causal, - .softmax_scale = softmax_scale, - .window_size_left = window_size_left, - .window_size_right = window_size_right, - .softcap = 0.0, - .is_rotary_interleaved = false, - .num_splits = 0, - .sm_margin = 0, - .cp_world_size = 1, - .cp_rank = 0, - }; - - flashattn.fa3_mha_fwd( - &q.toFlashattnTensor(), - &paged_k.toFlashattnTensor(), - &paged_v.toFlashattnTensor(), - &o.toFlashattnTensor(), - &cu_seqlens_q.toFlashattnTensor(), - null, - null, - &seqused_k.toFlashattnTensor(), - &block_table.toFlashattnTensor(), - null, - null, - null, - &softmax_lse.toFlashattnTensor(), - &softmax_lse_accum.toFlashattnTensor(), - &out_accum.toFlashattnTensor(), - &scheduler_metadata.toFlashattnTensor(), - null, - null, - ¶ms, - stream, - ); - - return null; - } - }; - - pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = opts; // autofix - stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); - stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); - stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); - - const o = switch (parameters) { - .decode => |decode_parameters| b: { - const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); - - const o = zml.ops.customCall( - Decode.custom_call_name, - .{ - q, - k_cache, - v_cache, - decode_parameters.cu_seqlens_q, - decode_parameters.seqused_k, - decode_parameters.block_table, - softmax_lse, - softmax_lse_accum, - decode_parameters.metadata.out_accum, - scheduler_metadata, - layer_index, - }, - .{q.shape()}, - .{ - .is_causal = true, - .max_seqlen_k = context.max_seqlen_k, - }, - .{ .has_side_effect = false }, - ); - - break :b o; - }, - .mixed => |mixed_parameters| b: { - const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); - const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); - var o = zml.ops.customCall( - Prefill.custom_call_name, - .{ - q, - k_cache, - v_cache, - mixed_parameters.cu_seqlens_q_prefill, - mixed_parameters.seqused_k_prefill, - mixed_parameters.block_table_prefill, - softmax_lse_prefill, - softmax_lse_accum_prefill, - mixed_parameters.metadata.out_accum, - scheduler_metadata_prefill, - mixed_parameters.metadata.host_metadata, - layer_index, - }, - .{q.shape()}, - .{ - .is_causal = false, - .max_seqlen_k = context.max_seqlen_k, - }, - .{ .has_side_effect = false }, - ); - - const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); - const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); - const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); - const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); - var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); - - const o_decode = zml.ops.customCall( - Decode.custom_call_name, - .{ - q_decode, - k_cache, - v_cache, - mixed_parameters.cu_seqlens_q_decode, - mixed_parameters.seqused_k_decode, - mixed_parameters.block_table_decode, - softmax_lse_decode, - softmax_lse_accum_decode, - mixed_parameters.metadata.out_accum, - scheduler_metadata_decode, - layer_index, - }, - .{q_decode.shape()}, - .{ - .is_causal = true, - .max_seqlen_k = context.max_seqlen_k, - }, - .{ .has_side_effect = false }, - ); - - o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); - break :b o; - }, - }; - - return o; - } -}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig deleted file mode 100644 index deb63ec7..00000000 --- a/zml/attention/paged_attention.zig +++ /dev/null @@ -1,145 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../zml.zig"); -const flashattn = @import("flashattn.zig"); - -const PagedAttention = @This(); - -pub const Backend = enum { - cuda_fa2, - cuda_fa3, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => b: { - const first_device = platform.devices[0].pjrt_device; - - if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { - if (std.mem.eql(u8, cc, "9.0")) { - break :b .cuda_fa3; - } - } - - break :b .cuda_fa2; - }, - else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), - }; - } -}; - -pub const Options = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Options, - cuda_fa3: flashattn.paged_fa3.Options, - - pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { - return switch (backend) { - .cuda_fa2 => if (is_prefill) .{ - .cuda_fa2 = .{ - .mixed = .{ - .batch_size_decode = batch_size, - .batch_size_prefill = batch_size, - .max_num_pages = seq_len / page_chunk_size, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, - }, - }, - } else .{ - .cuda_fa2 = .{ - .decode = .{ - .batch_size = batch_size, - .max_num_pages = seq_len / page_chunk_size, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, - }, - }, - }, - .cuda_fa3 => if (is_prefill) .{ - .cuda_fa3 = .{ - .mixed = .{ - .batch_size_decode = batch_size, - .batch_size_prefill = batch_size, - .max_num_pages = seq_len / page_chunk_size, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, - }, - }, - } else .{ - .cuda_fa3 = .{ - .decode = .{ - .batch_size = batch_size, - .max_num_pages = seq_len / page_chunk_size, - .max_seqlen_k = seq_len, - .max_token_count = max_token_count, - .num_heads = num_attention_heads, - .head_dim = head_dim, - }, - }, - }, - }; - } - - pub fn isPrefill(self: Options) bool { - return switch (self) { - inline else => |v| v.isPrefill(), - }; - } -}; - -pub const Parameters = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Parameters, - cuda_fa3: flashattn.paged_fa3.Parameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, - .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, - .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - inline else => |v| v.allocationSize(), - }; - } -}; - -/// Internal state that can be used inside the model code. It's derived from Parameters and Option. -pub const Context = union(Backend) { - cuda_fa2: flashattn.paged_fa2.Context, - cuda_fa3: flashattn.paged_fa3.Context, - - pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { - return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, - .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, - }; - } -}; - -pub const AttentionOptions = struct { - is_causal: bool = true, - sliding_window: i32 = -1, -}; - -pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { - _ = k; // autofix - _ = v; // autofix - return switch (parameters) { - .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), - .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), - }; -} diff --git a/zml/buffer.zig b/zml/buffer.zig index 7b0a88ac..8714d80f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,43 +1,113 @@ const std = @import("std"); const pjrt = @import("pjrt"); +const platforms = @import("platforms"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); const log = std.log.scoped(.zml); +test { + std.testing.refAllDecls(Buffer); + std.testing.refAllDecls(Buffer.HostAccessible); +} + /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + _platform: *const Platform, _shape: Shape, - platform: *const Platform, _shards: Shards, - pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; + /// Whether executions may donate a buffer, writing their outputs into it. + /// `HostAccessible.init` uses it to decide whether devices share allocations. + pub const Donation = enum { + /// Executions only read the buffer. Unlike XLA's undonatable buffers, the + /// host may still rewrite a HostAccessible one between executions. + undonatable, + /// Executions may donate the buffer, replacing its handles with their outputs. + donatable, + }; + + pub const HostAccessible = @import("buffer/HostAccessible.zig"); + + pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + pub const Shard = struct { + _platform: *const Platform, + _pjrt_buffer: *pjrt.Buffer, + + pub fn devicePtr(self: *const Shard) *anyopaque { + return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } + }; + + pub const ShardIterator = struct { + _platform: *const Platform, + _shards: []const *pjrt.Buffer, + _index: usize = 0, + + pub fn remaining(self: *ShardIterator) usize { + return self._shards.len -| self._index; + } + + pub fn next(self: *ShardIterator) ?Shard { + defer self._index += 1; + if (self._index >= self._shards.len) return null; + + return .{ + ._pjrt_buffer = self._shards[self._index], + ._platform = self._platform, + }; + } + }; + pub const FromOptions = struct { wait: bool = true, memory: Memory.Kind = .default }; /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *Buffer) void { - // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { - buffer.deinit(self.platform.pjrt_api); + buffer.deinit(self._platform.pjrt_api); } + self._shards = .empty; + } + + /// Given a flat struct (static size, no slices) containing `zml.Buffer`, `deinit` each one of them. + pub fn deinitAll(T: type, buffers: *mem.Bufferized(T)) void { + meta.visitFlatStruct(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); + } + + /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. + /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, + /// This only impacts device memory. + pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { + meta.visit(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); } /// This Buffer shape. @@ -45,68 +115,90 @@ pub const Buffer = struct { return self._shape; } + pub fn numShards(self: Buffer) u32 { + return @intCast(self._shards.len); + } + + pub fn shards(self: *const Buffer) ShardIterator { + return .{ + ._platform = self._platform, + ._shards = self._shards.constSlice(), + }; + } + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); + const pl = Sharding.Placement.init(&self._shape) catch |err| { + return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); + }; + try writer.print("{f}", .{pl}); } /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(io: std.Io, platform: *const Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { - var res: Buffer = .{ - .platform = platform, - ._shape = shape_, - ._shards = .{}, + pub fn from( + io: std.Io, + platform: *const Platform, + shape_: Shape, + data_: []const u8, + opts: FromOptions, + ) !Buffer { + var pjrt_shards: Shards = .empty; + var packed_shape = shape_.packedShape(); + if (packed_shape._sharding.mesh == null) packed_shape._sharding.mesh = platform.replicated_mesh; + const byte_strides = packed_shape.computeByteStrides(); + const placement = Sharding.Placement.initWithByteStrides(&packed_shape, byte_strides.constSlice()) catch |err| { + @branchHint(.cold); + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ packed_shape, err }); + }; + const ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()); + const layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()); + + errdefer for (pjrt_shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); }; - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; - //const chunk_size = if (sharding_ax) |ax| cs: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); - // break :cs @divExact(shape_.dim(ax), n_partitions); - //} else 0; - //_ = chunk_size; + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + const byte_size: usize = if (packed_shape.rank() == 0) + packed_shape.dtype().sizeOf() + else + @intCast(packed_shape.dim(0) * byte_strides.get(0)); + stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); + const base_ptr = data_.ptr; - const buffer_type = pjrtx.bufferTypeFromDtype(shape_.dtype()); - const byte_strides = shape_.computeByteStrides(); - - const devices = platform.devices; - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - //const buf = if (sharding_ax) |ax| buf: { - // @panic("sharding not implemented"); - // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; - // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - //} else host_buffer; - - const args = pjrt.Client.BufferFromHostBufferArgs{ - .data = data_.ptr, - .buffer_type = buffer_type, - .dims = shape_.dims(), - .byte_strides = byte_strides.slice(), + for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { + const memory = platform.devices[device.id].memory(opts.memory).?; + const args: pjrt.Client.BufferFromHostBufferArgs = .{ + // Change for each device + .data = placement.shardPtr(device_index, base_ptr), + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices + .layout = layout, + .dims = placement.local_dims.constSlice(), + .buffer_type = ty, + .byte_strides = byte_strides.constSlice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = devices[i].memory(opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - if (event) |ev| ev.deinit(platform.pjrt_api); - res._shards.appendAssumeCapacity(pjrt_buffer); + + pjrt_shards.appendAssumeCapacity(pjrt_buffer); } if (opts.wait) { - try res.await(io); + try awaitShards(pjrt_shards.constSlice(), platform, io); } - return res; + return .{ ._platform = platform, ._shape = shape_, ._shards = pjrt_shards }; } /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { - return from(io, platform, sh, data, .{}); + return fromBytesOpts(io, platform, sh, data, .{}); + } + + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, data, opts); } /// Copies the given zml.Slice to the accelerator memory and @@ -122,80 +214,70 @@ pub const Buffer = struct { /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); + return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { - for (self._shards.constSlice()) |buffer| { - const ev = buffer.readyEvent(self.platform.pjrt_api); - defer ev.deinit(self.platform.pjrt_api); - try ev.await(self.platform.pjrt_api, io); + try awaitShards(self._shards.constSlice(), self._platform, io); + } + + fn awaitShards(pjrt_shards: []const *pjrt.Buffer, platform: *const Platform, io: std.Io) !void { + for (pjrt_shards) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try event.await(platform.pjrt_api, io); } } pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; - pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { - _ = io; // autofix - var res: Buffer = .{ - .platform = platform, + pub fn uninitialized( + _: std.Io, + platform: *const Platform, + shape_: Shape, + opts: UnitializedOptions, + ) !Buffer { + std.log.debug("uninitialized {f}", .{shape_}); + var buffer: Buffer = .{ + ._platform = platform, ._shape = shape_, - ._shards = .{}, + ._shards = .empty, }; - errdefer for (res._shards.slice()) |shard| { + const metadata: PjrtBufferMetadata = .init(platform, shape_); + errdefer for (buffer._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); - const n_partitions = platform.sharding().num_partitions; + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - //const shard_shape = if (sharding_ax) |ax| s: { - // // This kind of sharding error should be detected earlier on. - // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); - // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); - // break :s shard_shape; - //} else shape_; + for (platform.physical_mesh.devices_in_canonical_order) |device| { + const memory = platform.devices[device.id].memory(opts.memory).?; + const args: pjrt.Client.CreateUninitializedBufferArgs = .{ + // Change for each device + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices + .layout = metadata.layout, + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + }; - var args = pjrt.Client.CreateUninitializedBufferArgs{ - .dims = shape_.dims(), - .element_type = pjrtx.bufferTypeFromDtype(shape_.dtype()), - .layout = .{ - .tiled = .{ - .minor_to_major = constants.minorToMajor(shape_.rank()), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - // set per device, see below. - .dst = undefined, - }; - - for (platform.devices[0..n_partitions]) |*device| { - args.dst = .{ .memory = device.memory(opts.memory).pjrt_memory }; - const shard = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); - res._shards.appendAssumeCapacity(shard); + const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); + buffer._shards.appendAssumeCapacity(shard_buffer); } - return res; + return buffer; } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - .platform = platform, - ._shape = shape_, - ._shards = shards, - }; - } - pub fn devicePtr(self: Buffer) *anyopaque { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); - return self._shards.get(0).opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable; + return .{ + ._platform = platform, + ._shape = sh, + ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, + }; } /// Fetches the content of the given buffer into a stack variable of the given type. @@ -210,10 +292,22 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { - //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); - if (maybe_event) |event| { - try event.await(self.platform.pjrt_api, io); + stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); + + const placement = placementOrPanic(self._platform, self._shape); + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed + const sub_slice = placement.shardSlice(shard_index, slice); + if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; + + const size_bytes = placement.localShape(self._shape).byteSize(); + const destination = sub_slice.data()[0..size_bytes]; + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } } } @@ -223,8 +317,140 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - try self.toSlice(io, slice); + const placement = placementOrPanic(self._platform, self._shape); + + var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); + defer shard_slice.free(allocator); + + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + const sub_slice = placement.shardSlice(shard_index, slice); + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + + // TODO: why is this using Slice.copy while `toSlice` errors out ? + // TODO: why is this writing in a copy rather than directly in place. + sub_slice.copy(shard_slice.constData()); + } return slice; } + + /// The memory used by this Buffer across all devices + /// ie: `num_devices * shard_byte_size` + /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. + pub fn byteSize(self: Buffer) usize { + const placement = placementOrPanic(self._platform, self._shape); + return placement.localShape(self._shape).byteSize() * self._shards.len; + } + + pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { + return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } + + /// Creates a view of the given buffer + pub fn createView(self: Buffer) !Buffer { + const platform = self._platform; + + var view_shards_buffer: [MAX_NUM_SHARDS]*pjrt.Buffer = undefined; + var view_shard_list: std.ArrayList(*pjrt.Buffer) = .initBuffer(&view_shards_buffer); + errdefer for (view_shard_list.items) |shard| { + shard.deinit(platform.pjrt_api); + }; + + for (self._shards.constSlice()) |shard| { + const view_shard = try platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api), + .dims = shard.dimensions(platform.pjrt_api), + .element_type = shard.elementType(platform.pjrt_api), + .layout = try shard.memoryLayout(platform.pjrt_api), + .device = try shard.device(platform.pjrt_api), + }); + view_shard_list.appendAssumeCapacity(view_shard); + } + + return fromPjrtBuffers(platform, self._shape, view_shard_list.items); + } }; + +test "device round-trip" { + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); + + const x: [8][8]u32 = .{ + .{ 0, 1, 2, 3, 4, 5, 6, 7 }, + .{ 8, 9, 10, 11, 12, 13, 14, 15 }, + .{ 16, 17, 18, 19, 20, 21, 22, 23 }, + .{ 24, 25, 26, 27, 28, 29, 30, 31 }, + .{ 32, 33, 34, 35, 36, 37, 38, 39 }, + .{ 40, 41, 42, 43, 44, 45, 46, 47 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55 }, + .{ 56, 57, 58, 59, 60, 61, 62, 63 }, + }; + + const x_h: zml.Slice = .init(.withPartitioning( + .init(.{ .b = 8, .d = 8 }, .u32), + tp, + .{ .b = .model }, + ), std.mem.asBytes(&x)); + // no free: x_h is stack allocated + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); + try std.testing.expectEqual(platform.devices.len, x_d.numShards()); + + { + const x_h_reborn: zml.Slice = try x_d.toSliceAlloc(allocator, io); + defer x_h_reborn.free(allocator); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); + } + + { + var x_2: @TypeOf(x) = undefined; + const x_h_reborn: zml.Slice = .init(x_h.shape, std.mem.asBytes(&x_2)); + // no free: x_h_reborn is stack allocated + try x_d.toSlice(io, x_h_reborn); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); + } +} + +pub const PjrtBufferMetadata = struct { + shape: Shape, + ty: pjrt.BufferType, + placement: Sharding.Placement, + layout: pjrt.MemoryLayout, + + /// Inline because the layout may point to stack memory, as with + /// `Platform.defaultMemoryLayout`. Outside `Buffer` so that + /// `Buffer.HostAccessible` can use it without it being part of the API: + /// zml.zig only exports `Buffer` from this file. + pub inline fn init(platform: *const Platform, shape_: Shape) PjrtBufferMetadata { + + // Shards use the PJRT shape, which packs sub-byte elements. + const packed_shape = shape_.packedShape(); + const placement = placementOrPanic(platform, packed_shape); + + return .{ + .shape = packed_shape, + .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), + .placement = placement, + .layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()), + }; + } +}; + +fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(&sh) catch |err| { + @branchHint(.cold); + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); + }; +} diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig new file mode 100644 index 00000000..b3e82a9c --- /dev/null +++ b/zml/buffer/HostAccessible.zig @@ -0,0 +1,2202 @@ +//! A `Buffer` in `.host_pinned` memory that the host reads and writes in place +//! through views, which a plain `Buffer` placed there doesn't offer. Views +//! address elements as PJRT stores them, with sub-byte elements packed in bytes. +//! The host may only access the buffer between executions. Host pointers are +//! read at creation, and by `await` when an execution replaced the handles. +//! Executions must preserve shape, sharding and layout. +//! +//! Like `Buffer`, it is a value. Pass `buffer` to an execution to read it as +//! an input, or `&buffer` to donate it as an output, then `await` the +//! HostAccessible whose `buffer` the execution replaced. Copies share the +//! mapping, and a copy whose handles the execution did not replace panics on +//! `view` rather than write to an allocation the execution may have released. +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Buffer = @import("../buffer.zig").Buffer; +const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const testing = @import("../testing.zig"); + +const HostAccessible = @This(); + +/// The device side, with one PJRT buffer per device, for executions to read +/// or donate. +buffer: Buffer, +/// What views read, heap allocated so views can borrow it while the +/// HostAccessible moves. +mapping: *Mapping, + +/// The host side of the buffer. Views and blocks copy nothing from it: the +/// layout and shard placement are prepared once at creation and take several +/// KB, and copying them into every view, slice and block made memcpy most of +/// llmd's decode thread. +const Mapping = struct { + /// The PJRT shape, which packs sub-byte elements in bytes, unlike + /// `buffer._shape`. Views address its elements. + packed_shape: Shape, + shard_shape: Shape, + layout_metadata: LayoutMetadata, + allocations: HostPinnedAllocations, + /// Every shard holds the whole array densely, so element `i` sits at + /// `i * element_size` in each of them, as in replicated buffers on every + /// platform but TPU. `get`, `set` and blocks then skip locating elements + /// through the origins and the layout. + contiguous: bool, + + /// Takes ownership of the allocations, on success only. + fn create( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + allocations: HostPinnedAllocations, + ) !*Mapping { + const mapping = try allocator.create(Mapping); + errdefer allocator.destroy(mapping); + + const layout_metadata: LayoutMetadata = .init(shape_, shard_shape, layout); + + mapping.* = .{ + .packed_shape = shape_, + .shard_shape = shard_shape, + .layout_metadata = layout_metadata, + .allocations = allocations, + .contiguous = layout_metadata.addressing == .row_major and + allocations.regions.len == 1 and + shard_shape.count() == shape_.count(), + }; + + return mapping; + } +}; + +/// The host allocations behind the buffer's shards. Devices that share an +/// allocation map to one entry, and entries holding the same region of the +/// array are its replicas. Entries stay in region order, so a region's +/// replicas are a range of every field. +const HostPinnedAllocations = struct { + entries: std.MultiArrayList(Allocation) = .empty, + /// The regions the entries hold, in order of first appearance. + regions: stdx.BoundedArray(Region, Platform.MAX_NUM_DEVICES) = .empty, + + const empty: HostPinnedAllocations = .{}; + + const Allocation = struct { + /// Global coordinates of the allocation's first element. + origin: Coordinates, + /// Read from `handle`. Blocks borrow their region's range. + ptr: [*]u8, + /// The shard owning the allocation, which must destroy it: an index + /// into `buffer._shards`, whose handles execution may replace. + shard_index: usize, + /// The handle `ptr` was read from. Execution replaces the handles of + /// donated buffers with its outputs, which may not reuse the allocation. + handle: *pjrt.Buffer, + /// Whether `handle` holds an external reference, which keeps the + /// allocation alive for the device views of other devices. + shared: bool = false, + }; + + /// `entries[start..][0..len]` hold the region's replicas, in the order they + /// were added. Reads use the first. + const Region = struct { start: u8, len: u8 }; + + fn deinit(self: *HostPinnedAllocations, allocator: std.mem.Allocator) void { + self.entries.deinit(allocator); + } + + /// Add an allocation to the replicas of its region, within the + /// reserved capacity. + fn add(self: *HostPinnedAllocations, shard_shape: Shape, allocation: Allocation) void { + const region_index = self.findRegion(allocation.origin, shard_shape) orelse { + self.regions.appendAssumeCapacity(.{ .start = @intCast(self.entries.len), .len = 1 }); + self.entries.appendAssumeCapacity(allocation); + return; + }; + const region = &self.regions.slice()[region_index]; + // Keep each region's entries together by shifting the regions after it. + self.entries.insertAssumeCapacity(region.start + region.len, allocation); + region.len += 1; + for (self.regions.slice()[region_index + 1 ..]) |*next| next.start += 1; + } + + /// The index of the region holding the shards that start at `origin`. + fn findRegion(self: *const HostPinnedAllocations, origin: Coordinates, shard_shape: Shape) ?usize { + const origins = self.entries.items(.origin); + for (self.regions.constSlice(), 0..) |region, i| { + if (origin.eqlWithin(origins[region.start], shard_shape)) return i; + } + return null; + } + + /// The host pointers of a region's replicas. + fn replicas(self: *const HostPinnedAllocations, region: Region) []const [*]u8 { + return self.entries.items(.ptr)[region.start..][0..region.len]; + } + + /// Read the host pointers of the allocations from their current handles. + /// Each read takes and releases a hold on the PJRT buffer and allocates an + /// external reference, about 0.9 us: llmd would make thousands per step if + /// views read them, so they are read at creation and by `await` only. + fn refreshPointers(self: *HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) !void { + const shard_indices = self.entries.items(.shard_index); + // Read every pointer before replacing any, so a failure keeps the previous ones. + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + for (shard_indices, ptrs[0..shard_indices.len]) |shard_index, *ptr| { + ptr.* = @ptrCast(try handles[shard_index].opaqueDeviceMemoryDataPointer(api)); + } + @memcpy(self.entries.items(.ptr), ptrs[0..shard_indices.len]); + for (self.entries.items(.handle), shard_indices) |*handle, shard_index| handle.* = handles[shard_index]; + } + + /// Whether the host pointers were read from the current handles, which + /// execution replaces for donated buffers. + fn pointersAreCurrent(self: *const HostPinnedAllocations, handles: []const *pjrt.Buffer) bool { + // Host memory in tests has no handles: its pointers never change. + if (handles.len == 0) return true; + for (self.entries.items(.handle), self.entries.items(.shard_index)) |handle, shard_index| { + if (handle != handles[shard_index]) return false; + } + return true; + } + + /// Destroy every handle, the device views before the allocations they borrow. + fn deinitHandles(self: *const HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) void { + const shard_indices = self.entries.items(.shard_index); + for (handles, 0..) |handle, index| { + if (std.mem.indexOfScalar(usize, shard_indices, index) == null) handle.deinit(api); + } + for (shard_indices, self.entries.items(.shared)) |shard_index, shared| { + const handle = handles[shard_index]; + // `shared` is set by the one successful increase, and only undonatable + // buffers share allocations, whose handles execution never replaces. + if (shared) handle.decreaseExternalReferenceCount(api) catch |err| { + std.debug.panic("shared HostAccessible allocation has no reference to release, was it donated? {}", .{err}); + }; + handle.deinit(api); + } + } +}; + +/// Host memory standing in for PJRT buffers, to test layouts and shardings +/// that no available platform produces: every platform but TPU uses dense +/// row-major layouts, and CI has few devices. `origins` holds the row-major +/// index in `shape_` of each allocation's first element, and `ptrs` one pointer +/// per origin. Copies the layout, the origins and the pointers, and borrows the +/// storage. +pub fn initForTests( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + origins: []const usize, + ptrs: []const [*]u8, +) !HostAccessible { + if (!builtin.is_test) @compileError("HostAccessible.initForTests is only available in tests"); + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, origins.len); + // Without PJRT buffers, there are no handles to read pointers from or destroy. + for (origins, ptrs) |origin, ptr| { + allocations.add( + shard_shape, + .{ + // An empty array has no element to locate the origin of. + .origin = if (shape_.count() == 0) .zero else .unflatten(shape_, origin), + .ptr = ptr, + .shard_index = undefined, + .handle = undefined, + }, + ); + } + + return .{ + .buffer = .{ ._platform = undefined, ._shape = shape_, ._shards = .empty }, + .mapping = try .create(allocator, shape_, shard_shape, layout, allocations), + }; +} + +/// With `.undonatable`, devices holding the same shard share one allocation on +/// CPU, CUDA and ROCm: the first allocates it, and the others read it through +/// device views. Elsewhere each device owns an allocation, and writes reach +/// every one of them. Execution never replaces the handles of an undonatable +/// buffer, so `await` never reads its host pointers again. +/// With `.donatable`, each device owns an allocation: writing a shared one +/// would expose the output of a device to the reads of another. +pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + shape_: Shape, + donation: Buffer.Donation, +) !HostAccessible { + const can_share_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => donation == .undonatable, + else => false, + }; + + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); + const devices = platform.physical_mesh.devices_in_canonical_order; + const api = platform.pjrt_api; + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, devices.len); + errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); + + for (devices, 0..) |device, shard_index| { + const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); + const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; + + if (shared_region) |region_index| { + // Devices sharing allocations add no replicas: the region has a single entry. + const entry = allocations.regions.get(region_index).start; + const existing = allocations.entries.get(entry); + if (!existing.shared) { + try existing.handle.increaseExternalReferenceCount(api); + allocations.entries.items(.shared)[entry] = true; + } + buffer._shards.appendAssumeCapacity(try platform.pjrt_client.createViewOfDeviceBuffer(api, .{ + .device_buffer_ptr = try existing.handle.opaqueDeviceMemoryDataPointer(api), + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + .layout = try existing.handle.memoryLayout(api), + // The backing host allocation is accessible across devices + // (portable pinned memory on CUDA/ROCm). This memory space + // associates the view with the current device rather than + // the device that originally allocated the memory. + .memory = memory, + })); + } else { + const handle = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + .layout = metadata.layout, + .dst = .{ .memory = memory }, + }); + buffer._shards.appendAssumeCapacity(handle); + // The pointer is read once the creation completes, after `await` below. + allocations.add(metadata.placement.localShape(metadata.shape), .{ + .origin = origin, + .ptr = undefined, + .shard_index = shard_index, + .handle = handle, + }); + } + } + + try buffer.await(io); + try allocations.refreshPointers(api, buffer._shards.constSlice()); + + return .{ + .buffer = buffer, + .mapping = try .create(allocator, metadata.shape, metadata.placement.localShape(metadata.shape), metadata.layout, allocations), + }; +} + +/// Execution must be done with the buffer. Copies must not be used after. +pub fn deinit(self: *const HostAccessible, allocator: std.mem.Allocator) void { + // Host memory in tests has neither handles nor a platform. + if (self.buffer._shards.len != 0) self.mapping.allocations.deinitHandles(self.buffer._platform.pjrt_api, self.buffer._shards.constSlice()); + self.mapping.allocations.deinit(allocator); + allocator.destroy(self.mapping); +} + +/// Wait for execution to finish, and read the host pointers of handles that +/// execution replaced. Take new views to access the output. +pub fn await(self: *HostAccessible, io: std.Io) !void { + try self.buffer.await(io); + const handles = self.buffer._shards.constSlice(); + const allocations = &self.mapping.allocations; + if (!allocations.pointersAreCurrent(handles)) try allocations.refreshPointers(self.buffer._platform.pjrt_api, handles); +} + +/// Borrows the mapping. Executions using the buffer must have completed, and +/// `await` must follow any that replaced the handles: `view` panics before +/// then rather than write to an allocation the execution may have released. +/// The view and its blocks are valid until the next execution starts or the +/// HostAccessible is deinitialized. +pub fn view(self: *const HostAccessible, comptime T: type) View(T) { + const mapping = self.mapping; + std.debug.assert(@sizeOf(T) == mapping.shard_shape.dtype().sizeOf()); + stdx.debug.assert(mapping.allocations.pointersAreCurrent(self.buffer._shards.constSlice()), "await the HostAccessible after execution replaces its handles", .{}); + return .{ .mapping = mapping, .len = mapping.packed_shape.count() }; +} + +/// Prints every element, read through a view: the same rules as `view` apply. +pub fn format(self: *const HostAccessible, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("HostAccessible({f}) {{", .{self.mapping.packed_shape}); + switch (self.mapping.shard_shape.dtype()) { + inline else => |dt| { + if (comptime dt.bitSizeOf() < 8) unreachable; + const values = self.view(dt.toZigType()); + var blocks = values.blocks(); + while (blocks.next()) |block| { + for (block.firstReplica()) |value| try writer.print(" {any}", .{value}); + } + }, + } + try writer.writeAll(" }"); +} + +/// A range of the array's elements in row-major order, borrowed from a +/// HostAccessible. Reads use the first replica, and writes reach every replica. +pub fn View(comptime T: type) type { + return struct { + mapping: *const Mapping, + start: usize = 0, + len: usize, + + const Self = @This(); + + /// A view without elements, which never accesses the buffer. + pub const empty: Self = .{ + .mapping = undefined, + .len = 0, + }; + + /// Select `len_` elements starting at `start`, or the rest of the view for null. + pub fn slice(self: Self, start: usize, len_: ?usize) Self { + std.debug.assert(start <= self.len); + const len = len_ orelse self.len - start; + std.debug.assert(len <= self.len - start); + var result = self; + result.start += start; + result.len = len; + return result; + } + + // Single elements are addressed directly, without slicing and iterating. + pub fn get(self: Self, index: usize) T { + const element = self.locateElement(index); + return itemAt(self.mapping.allocations.replicas(element.region)[0], element.byte_offset).*; + } + + pub fn set(self: Self, index: usize, value: T) void { + const element = self.locateElement(index); + for (self.mapping.allocations.replicas(element.region)) |ptr| itemAt(ptr, element.byte_offset).* = value; + } + + fn locateElement(self: *const Self, index: usize) struct { region: HostPinnedAllocations.Region, byte_offset: usize } { + std.debug.assert(index < self.len); + if (self.mapping.contiguous) { + return .{ .region = self.mapping.allocations.regions.get(0), .byte_offset = (self.start + index) * @sizeOf(T) }; + } + const location = self.locate(.unflatten(self.mapping.packed_shape, self.start + index)); + return .{ .region = location.region, .byte_offset = self.mapping.layout_metadata.byteOffset(location.offset, location.local) }; + } + + pub fn fill(self: Self, value: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memset(block.replica(i), value); + } + } + + /// Fill with `first + i * step`, where `i` is relative to this view. + /// `T` must be an integer type holding every `i` and value; safe builds check it. + pub fn fillIota(self: Self, first: T, step: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + // TODO: SIMD + for (items, block.offset..) |*value, i| value.* = first + @as(T, @intCast(i)) * step; + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + pub fn copyFrom(self: Self, values: []const T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memcpy(block.replica(i), values[block.offset..][0..block.len]); + } + } + + pub fn copyTo(self: Self, values: []T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + @memcpy(values[block.offset..][0..block.len], block.firstReplica()); + } + } + + /// The index of the first element equal to `value`, searching each + /// block with `std.mem.indexOfScalar`. + pub fn indexOfScalar(self: Self, value: T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfScalar(T, block.firstReplica(), value)) |index| return block.offset + index; + } + return null; + } + + /// The index of the first element equal to any of `values`, searching + /// each block with `std.mem.indexOfAny`. + pub fn indexOfAny(self: Self, values: []const T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfAny(T, block.firstReplica(), values)) |index| return block.offset + index; + } + return null; + } + + /// Source and destination must not overlap. + pub fn copyFromView(self: Self, src: Self) void { + std.debug.assert(src.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + src.slice(block.offset, block.len).copyTo(items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + /// The elements as one read-only slice, for reads that index or zip + /// several views. Contiguous buffers, such as replicated ones on every + /// platform but TPU, return their first replica in place; others + /// copy into `scratch`, which must hold `len` elements on every + /// platform. The slice is valid until the next execution, and + /// writes must go through the view. + pub fn constSlice(self: Self, scratch: []T) []const T { + std.debug.assert(scratch.len >= self.len); + if (self.inPlace()) |block| return block.firstReplica(); + self.copyTo(scratch[0..self.len]); + return scratch[0..self.len]; + } + + /// A slice to write the elements into, then `commit`, for writes + /// that compute elements in place. Contiguous buffers hand out their + /// first replica; others hand out `scratch`, which must hold `len` + /// elements on every platform. + pub fn edit(self: Self, scratch: []T) Edit { + std.debug.assert(scratch.len >= self.len); + return .{ + .view = self, + .items = if (self.inPlace()) |block| block.firstReplica() else scratch[0..self.len], + }; + } + + pub const Edit = struct { + view: Self, + /// Holds the previous values or garbage: write every element. + items: []T, + + /// Make the writes reach every replica: copy the first replica to + /// the others, or `items` into the view. + pub fn commit(self: Edit) void { + const block = self.view.inPlace() orelse return self.view.copyFrom(self.items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), self.items); + } + }; + + /// The view as a single block, when the buffer is contiguous. + fn inPlace(self: Self) ?Block { + if (self.len == 0 or !self.mapping.contiguous) return null; + var iterator = self.blocks(); + return iterator.next().?; + } + + /// Visit runs of consecutive logical elements in global row-major + /// order, without padding, grouping replicas of each logical range. + /// Shards must cover the view with disjoint regions or exact replicas. + pub fn blocks(self: Self) BlockIterator { + return .init(self); + } + + /// Visit the same blocks in reverse order. Elements within each + /// replica slice remain in ascending logical order. + pub fn reverseBlocks(self: Self) ReverseBlockIterator { + return .init(self); + } + + /// A run of elements that are consecutive in the view's row-major order and + /// contiguous in memory, so each replica holds them as a plain slice. + /// Writes must reach every replica; reads use the first. Borrows the buffer's + /// host pointers, like the view it comes from. + pub const Block = struct { + /// Logical position of the first element, relative to this view. + offset: usize, + len: usize, + /// Byte offset of the first element in the allocation of every replica. + byte_offset: usize, + /// Host pointer of each replica's allocation. + replicas: []const [*]u8, + + /// The block's elements in its `i`-th replica. + pub fn replica(self: Block, i: usize) []T { + const ptr: [*]T = @ptrCast(@alignCast(self.replicas[i] + self.byte_offset)); + return ptr[0..self.len]; + } + + /// The elements in the first replica, which reads use. + pub fn firstReplica(self: Block) []T { + return self.replica(0); + } + }; + + pub const BlockIterator = OrderedBlockIterator(false); + pub const ReverseBlockIterator = OrderedBlockIterator(true); + + fn OrderedBlockIterator(comptime reverse: bool) type { + return struct { + view: Self, + /// Next logical offset, or one past it for reverse traversal. + offset: usize, + /// Coordinates of the next block's first (or reverse last) element. + global: Coordinates, + + const Iterator = @This(); + + fn init(v: Self) Iterator { + return .{ + .view = v, + .offset = if (reverse) v.len else 0, + .global = if (v.len == 0 or v.mapping.contiguous) .zero else .unflatten( + v.mapping.packed_shape, + v.start + (if (reverse) v.len - 1 else 0), + ), + }; + } + + pub fn next(self: *Iterator) ?Block { + const v = &self.view; + if (if (reverse) self.offset == 0 else self.offset == v.len) return null; + if (v.mapping.contiguous) { + // The whole view is one run in every replica. + self.offset = if (reverse) 0 else v.len; + return v.blockAt(v.mapping.allocations.regions.get(0), 0, v.len, v.start * @sizeOf(T)); + } + const location = v.locate(self.global); + const bounds = v.blockBounds(location.offset); + const len = if (reverse) + @min(location.offset + 1 - bounds.start, self.offset) + else + @min(bounds.end - location.offset, v.len - self.offset); + // A block is physically contiguous, so its last element's + // address also gives its start without decoding coordinates again. + const byte_offset = v.mapping.layout_metadata.byteOffset(location.offset, location.local) - + (if (reverse) (len - 1) * v.mapping.layout_metadata.element_size else 0); + const offset = if (reverse) self.offset - len else self.offset; + self.offset = if (reverse) offset else offset + len; + if (if (reverse) self.offset != 0 else self.offset != v.len) { + self.global.move(reverse, v.mapping.packed_shape, len); + } + return v.blockAt(location.region, offset, len, byte_offset); + } + }; + } + + const Bounds = struct { start: usize, end: usize }; + + /// The block containing a shard-local element: elements contiguous in both + /// physical memory and global row-major order. Bounds are shard-local + /// row-major indices, not physical memory offsets. + /// + /// Example: an unsharded [2,5] array with row-major 2x2 tiles. + /// + /// 1. Logical indices, with tile boundaries. `_` denotes padding: + /// ```text + /// tile 0 tile 1 tile 2 + /// +-----+ +-----+ +-----+ + /// | 0 1 | | 2 3 | | 4 _ | + /// | 5 6 | | 7 8 | | 9 _ | + /// +-----+ +-----+ +-----+ + /// ``` + /// 2. Tiles are stored consecutively, each in row-major order: + /// ```text + /// [0 1 5 6] [2 3 7 8] [4 _ 9 _] + /// ``` + /// Thus 0 and 1 are adjacent in memory, but 1 and 2 are not. + /// `block_size = 2` is the maximum contiguous logical run. + /// + /// 3. Align runs within each logical row, clipping before padding: + /// ```text + /// last_dim_size = 5 + /// alignment_size = max(5, 2) = 5 + /// + /// alignment_start blocks of logical elements row end + /// 0 [0 1] [2 3] [4] 5 + /// 5 [5 6] [7 8] [9] 10 + /// actual block sizes: 2 2 1 + /// + /// local alignment_start start end returned bounds + /// 4 0 4 5 [4,5) + /// 5 5 5 7 [5,7) + /// 6 5 5 7 [5,7) + /// 9 5 9 10 [9,10) + /// ``` + /// For local=6, start = 6 - (6-5)%2 = 5, end = min(5+2, 5+5) = 7. + /// For local=4, end = min(4+2, 0+5) = 5: the final run is shortened. + /// + /// Without tiles, the same dense row-major array has block_size=10 + /// and alignment_size=10: one block [0,10) spanning both rows. + fn blockBounds(self: *const Self, local: usize) Bounds { + const block_size = self.mapping.layout_metadata.block_size; + // Runs shorter than a row align within that row. Larger runs + // span a whole number of rows, so alignment uses the run itself. + const alignment_size = self.mapping.layout_metadata.alignment_size; + const alignment_start = local - local % alignment_size; + // Round the position relative to alignment_start down to a run boundary. + const start = local - (local - alignment_start) % block_size; + // A row's final run may be shorter; stop at the alignment interval's end. + return .{ .start = start, .end = @min(start + block_size, alignment_start + alignment_size) }; + } + + fn blockAt(self: *const Self, region: HostPinnedAllocations.Region, offset: usize, len: usize, byte_offset: usize) Block { + return .{ + .offset = offset, + .len = len, + .byte_offset = byte_offset, + .replicas = self.mapping.allocations.replicas(region), + }; + } + + fn itemAt(ptr: [*]u8, byte_offset: usize) *T { + return @ptrCast(@alignCast(ptr + byte_offset)); + } + + const Location = struct { + region: HostPinnedAllocations.Region, + /// Coordinates of the element within its shard. + local: Coordinates, + /// Row-major index of the element within its shard. + offset: usize, + }; + + /// Locate the region holding an element, retaining its shard-local + /// coordinates for physical addressing. + fn locate(self: *const Self, global: Coordinates) Location { + const origins = self.mapping.allocations.entries.items(.origin); + for (self.mapping.allocations.regions.constSlice()) |region| { + const origin = origins[region.start]; + var local: Coordinates = .zero; + var offset: usize = 0; + for (self.mapping.shard_shape.dims(), 0..) |dim_, axis| { + const dim: usize = @intCast(dim_); + if (global.axes[axis] < origin.axes[axis] or global.axes[axis] - origin.axes[axis] >= dim) break; + local.axes[axis] = global.axes[axis] - origin.axes[axis]; + offset = offset * dim + local.axes[axis]; + } else { + return .{ .region = region, .local = local, .offset = offset }; + } + } + @panic("The union of all regions should cover the global shape."); + } + }; +} + +/// A position in an array, one coordinate per axis. Axes past the rank stay zero. +const Coordinates = struct { + axes: [Shape.MAX_RANK]usize, + + const zero: Coordinates = .{ .axes = @splat(0) }; + + /// The global coordinates of the first element of a device's shard. + fn ofShard(placement: *const Sharding.Placement, device_index: usize, shape: Shape) Coordinates { + // TODO: derive shard coordinates directly from DeviceGrid instead of reconstructing slices. + var result: Coordinates = .zero; + for (placement.slices(device_index, shape).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); + return result; + } + + /// The coordinates of the element at `offset` of `shape`, in row-major order. + fn unflatten(shape_: Shape, offset: usize) Coordinates { + var result: Coordinates = .zero; + var remaining = offset; + var axis = shape_.rank(); + while (axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + result.axes[axis] = remaining % dim; + remaining /= dim; + } + return result; + } + + /// Whether the coordinates match on the axes of `shape`. + fn eqlWithin(self: Coordinates, other: Coordinates, shape_: Shape) bool { + return std.mem.eql(usize, self.axes[0..shape_.rank()], other.axes[0..shape_.rank()]); + } + + /// Advance in row-major order, decoding only axes that carry or borrow. + /// The resulting coordinates must still be within the shape. + fn move(self: *Coordinates, comptime reverse: bool, shape_: Shape, count: usize) void { + const axes = &self.axes; + var carry = count; + var axis = shape_.rank(); + while (carry != 0 and axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + if (reverse) { + if (carry <= axes[axis]) { + axes[axis] -= carry; + return; + } + carry -= axes[axis] + 1; + axes[axis] = dim - 1 - carry % dim; + } else { + const available = dim - axes[axis]; + if (carry < available) { + axes[axis] += carry; + return; + } + carry -= available; + axes[axis] = carry % dim; + } + carry = carry / dim + 1; + } + std.debug.assert(carry == 0); + } +}; + +/// Physical addressing and logical block boundaries prepared once per HostAccessible. +/// All metadata is stored by value; the source layout is needed only during init. +const LayoutMetadata = struct { + rank: u4, + element_size: usize, + block_size: usize, + alignment_size: usize, + addressing: Addressing, + + const Addressing = union(enum) { + row_major, + strided: ByteStrides, + tiled: Tiled, + + const ByteStrides = [Shape.MAX_RANK]usize; + + const Tiled = struct { + /// Unit and padding dimensions always hold index 0 and are left out. + digits: stdx.BoundedArray(Digit, Shape.MAX_RANK + pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// One physical dimension of a tiled layout, as a digit of one coordinate. + const Digit = struct { + byte_stride: usize, + /// Divides the coordinate, unless `div_shift` replaces it for a power of two. + divisor: usize, + /// A mask when `is_mask` is set; otherwise a general modulus. + modulus: usize, + axis: u3, + div_shift: ?std.math.Log2Int(usize), + is_mask: bool, + + fn init(axis: u3, dim: TiledDims.Dim, byte_stride: usize) Digit { + // Unbounded digits keep every bit of the quotient. + const modulus = if (dim.modulus == 0) std.math.maxInt(usize) else dim.modulus; + const is_mask = dim.modulus == 0 or std.math.isPowerOfTwo(modulus); + return .{ + .byte_stride = byte_stride, + .divisor = dim.divisor, + .modulus = if (is_mask and dim.modulus != 0) modulus - 1 else modulus, + .axis = axis, + .div_shift = if (std.math.isPowerOfTwo(dim.divisor)) @intCast(@ctz(dim.divisor)) else null, + .is_mask = is_mask, + }; + } + + fn value(self: Digit, coordinate: usize) usize { + const quotient = if (self.div_shift) |shift| coordinate >> shift else coordinate / self.divisor; + return if (self.is_mask) quotient & self.modulus else quotient % self.modulus; + } + }; + }; + }; + + fn init(shape: Shape, shard_shape: Shape, layout: pjrt.MemoryLayout) LayoutMetadata { + const last_dim_size: usize = if (shard_shape.rank() == 0) 1 else @intCast(shard_shape.dim(-1)); + const element_size: usize = shard_shape.dtype().sizeOf(); + + // Longest stretch where stepping to the next local index also steps to the next element in memory. + // Suppose a [2,5] shard tiled by (2,2): tiles are stored one after another, each row-major. + // With local indices the memory holds: + // `0 1 5 6 | 2 3 7 8 | 4 _ 9 _` + // 0 and 1 are adjacent but 1 and 2 aren't, so the run is 2. A row-major layout is a single run. + const addressing: Addressing, const physical_run: usize = addr: switch (layout) { + .strides => |strides| { + std.debug.assert(strides.byte_strides.len == shard_shape.rank()); + + const row_major = shard_shape.computeByteStrides(); + if (std.mem.eql(i64, strides.byte_strides, row_major.constSlice())) break :addr .{ .row_major, shard_shape.count() }; + + var byte_strides: Addressing.ByteStrides = @splat(0); + for (strides.byte_strides, 0..) |stride, axis| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_strides[axis] = @intCast(stride); + } + + const physical_run = + // Rows padded to an alignment keep their elements adjacent: a run is a row. + if (byte_strides[shard_shape.rank() - 1] == element_size) last_dim_size + // Permuted axes, e.g. a column-major [3,4] f32 with strides [4,12], put the + // innermost axis elsewhere in memory, so no two elements of a row are adjacent. + else 1; + break :addr .{ .{ .strided = byte_strides }, physical_run }; + }, + .tiled => |tiled| { + stdx.debug.assert(tiled.minor_to_major.len == shard_shape.rank(), "layout rank {} doesn't match shape rank {}", .{ tiled.minor_to_major.len, shard_shape.rank() }); + + if (tiled.tile_dims.len == 0 and isMinorToMajorRowMajor(tiled.minor_to_major, shard_shape.rank())) break :addr .{ .row_major, shard_shape.count() }; + + // The expanded dimensions are laid out row-major. + const expanded: TiledDims = .init(shard_shape, tiled); + var tiled_addressing: Addressing.Tiled = .{}; + var byte_stride = element_size; + // From the most minor physical dimension, the run grows while each + // dimension is the next digit of the innermost axis. + const last_axis = shard_shape.rank() -| 1; + var physical_run: usize = 1; + var run_grows = true; + var i = expanded.dims.len; + while (i > 0) { + i -= 1; + const dim = expanded.dims.get(i); + if (dim.size > 1) { + if (dim.axis) |axis| tiled_addressing.digits.appendAssumeCapacity(.init(axis, dim, byte_stride)); + if (run_grows) { + const next_digit = if (dim.axis) |axis| axis == last_axis and dim.divisor == physical_run else false; + if (!next_digit) { + run_grows = false; + } else if (dim.modulus == 0) { + // An unbounded digit holds the rest of the axis. + physical_run = last_dim_size; + run_grows = false; + } else { + physical_run *= dim.modulus; + // Padding after the digit's last value ends the run. + run_grows = dim.size == dim.modulus; + } + } + } + byte_stride *= dim.size; + } + physical_run = @min(physical_run, last_dim_size); + + break :addr .{ .{ .tiled = tiled_addressing }, physical_run }; + }, + }; + + // Longest stretch where stepping to the next local index also steps to the next global index. + // Suppose a global [2,6] split into two [2,3] shards, each holding half the columns. + // With global indices the shards hold: + // - shard 1: `0 1 2 | 6 7 8` + // - shard 2: `3 4 5 | 9 10 11` + // 2 and 6 are adjacent in shard 1 but not globally, so the run is 3. + const shard_run = run: { + var run: usize = 1; + var axis = shard_shape.rank(); + while (axis > 0) { + axis -= 1; + run *= @intCast(shard_shape.dim(axis)); + if (shape.dim(axis) != shard_shape.dim(axis)) break; + } + break :run run; + }; + const block_size = if (shard_shape.count() == 0) 0 else @min(physical_run, shard_run); + + return .{ + .rank = shard_shape.rank(), + .element_size = element_size, + .block_size = block_size, + // See `View.blockBounds`. + .alignment_size = @max(last_dim_size, block_size), + .addressing = addressing, + }; + } + + /// The byte offset in its shard of the element at shard-local row-major index + /// `local` and coordinates `coordinates`: row-major layouts use the index, + /// the others the coordinates. + fn byteOffset(self: *const LayoutMetadata, local: usize, coordinates: Coordinates) usize { + switch (self.addressing) { + .row_major => return local * self.element_size, + .strided => |*byte_strides| { + var offset: usize = 0; + for (coordinates.axes[0..self.rank], byte_strides[0..self.rank]) |coord, byte_stride| offset += coord * byte_stride; + return offset; + }, + .tiled => |*tiled| { + var offset: usize = 0; + for (tiled.digits.constSlice()) |digit| offset += digit.value(coordinates.axes[digit.axis]) * digit.byte_stride; + return offset; + }, + } + } +}; + +/// Whether a PJRT minor-to-major order lays the axes out row-major, like ZML shapes. +fn isMinorToMajorRowMajor(minor_to_major: []const i64, rank: usize) bool { + if (minor_to_major.len != rank) return false; + for (minor_to_major, 0..) |axis, i| { + if (axis != @as(i64, @intCast(rank - i - 1))) return false; + } + return true; +} + +/// The physical dimensions of a tiled layout, major to minor. XLA lays a tiled array +/// out row-major over these dimensions; see https://openxla.org/xla/tiled_layout. +const TiledDims = struct { + dims: stdx.BoundedArray(Dim, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// Every dimension holds the digit `(coordinate[axis] / divisor) % modulus`. + const Dim = struct { + /// Null for padding, which only holds index 0. + axis: ?u3, + size: usize, + divisor: usize = 1, + /// Zero for a digit without an upper bound, which holds `coordinate / divisor`. + modulus: usize = 0, + + /// The index of the tile along this dimension. + fn tileCount(self: Dim, tile: usize) Dim { + const size = std.math.divCeil(usize, self.size, tile) catch unreachable; + const axis = self.axis orelse return .{ .axis = null, .size = size }; + if (self.modulus == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile }; + if (self.modulus % tile == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile, .modulus = self.modulus / tile }; + // One tile holds every value of the digit. + return .{ .axis = null, .size = size }; + } + + /// The index within the tile along this dimension. + fn tileIndex(self: Dim, tile: usize) Dim { + const axis = self.axis orelse return .{ .axis = null, .size = tile }; + // XLA assumes nested tiles add no padding. + stdx.debug.assert( + self.modulus == 0 or self.modulus % tile == 0 or tile % self.modulus == 0, + "nested tile {} must divide, or be a multiple of, the {} values it tiles", + .{ tile, self.modulus }, + ); + if (self.modulus != 0 and self.modulus < tile) return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = self.modulus }; + return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = tile }; + } + }; + + /// Follows XLA's LayoutUtil::LinearIndexForNestedTiling. The physical dimensions + /// start in minor_to_major order. Each tile then splits the most minor ones, + /// padded with leading unit dimensions if the tile has more, into tile counts + /// followed by indices within the tile. Tile sizes are listed major to minor, so + /// a later tile splits the previous tile's indices and possibly its counts. + /// + /// Two consequences are easy to get wrong: + /// - `tile_dims[j]` of a k-dimensional tile belongs to `minor_to_major[k-1-j]`, + /// not to `minor_to_major[j]` nor to logical axis j. TPU's default + /// `{2,1,0:T(8,128)}` for rank-3 32-bit arrays tiles rows by 8 and columns + /// by 128. + /// - A later tile splits the previous tile's dimensions, not the grid of + /// tiles: `(8,128)(2,1)`, the bf16 packing, interleaves pairs of rows + /// within each tile. TPU tiles bf16 and 8-bit arrays this way. + fn init(shape: Shape, layout: pjrt.MemoryLayout.Tiled) TiledDims { + const rank = shape.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + var result: TiledDims = .{}; + var i = rank; + while (i > 0) { + i -= 1; + const axis: u3 = @intCast(layout.minor_to_major[i]); + result.dims.appendAssumeCapacity(.{ .axis = axis, .size = @intCast(shape.dim(axis)) }); + } + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (result.dims.len < tile_rank) result.dims.insert(0, .{ .axis = null, .size = 1 }) catch unreachable; + const start = result.dims.len - tile_rank; + for (tile, start..) |size_, d| { + // XLA marks dimensions combined with the next one by a negative size. + stdx.debug.assert(size_ > 0, "unsupported tile dimension {}", .{size_}); + const size: usize = @intCast(size_); + result.dims.appendAssumeCapacity(result.dims.get(d).tileIndex(size)); + result.dims.set(d, result.dims.get(d).tileCount(size)); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + return result; + } +}; + +// +// ============= +// === TESTS === +// ============= +// + +/// The byte offset of an element, computed from the layout directly to check `LayoutMetadata`. +fn referenceByteOffset(shape_: Shape, layout: pjrt.MemoryLayout, index: []const usize) usize { + if (!builtin.is_test) @compileError("referenceByteOffset is only available in tests"); + + return switch (layout) { + .tiled => |tiled| tiledElementOffset(shape_, tiled, index) * shape_.dtype().sizeOf(), + .strides => |strides| b: { + var byte_offset: i64 = 0; + for (index, strides.byte_strides) |coord, stride| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_offset += @as(i64, @intCast(coord)) * stride; + } + stdx.debug.assert(byte_offset >= 0, "destination byte offset must be non-negative, got {}", .{byte_offset}); + break :b @as(usize, @intCast(byte_offset)); + }, + }; +} + +/// Computes the element offset for one logical index in XLA/PJRT tiled layout order, +/// as XLA's LayoutUtil::LinearIndexForNestedTiling does. A direct port of that +/// function, kept independent of `TiledDims`: a reference sharing the prepared +/// model of tiling once let tests miss offsets that disagreed with XLA. +fn tiledElementOffset(shape_: Shape, layout: pjrt.MemoryLayout.Tiled, index: []const usize) usize { + if (!builtin.is_test) @compileError("tiledElementOffset is only available in tests"); + + const rank = shape_.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + + // Physical dimensions and indices, major to minor. + const Expanded = stdx.BoundedArray(usize, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS); + var dims: Expanded = .empty; + var indices: Expanded = .empty; + var i = rank; + while (i > 0) { + i -= 1; + const axis: usize = @intCast(layout.minor_to_major[i]); + dims.appendAssumeCapacity(@intCast(shape_.dim(axis))); + indices.appendAssumeCapacity(index[axis]); + } + + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (dims.len < tile_rank) { + dims.insert(0, 1) catch unreachable; + indices.insert(0, 0) catch unreachable; + } + // Tiled dimensions become tile counts, followed by the indices within the tile. + const start = dims.len - tile_rank; + for (tile, start..) |tile_dim_, d| { + const tile_dim: usize = @intCast(tile_dim_); + dims.appendAssumeCapacity(tile_dim); + indices.appendAssumeCapacity(indices.get(d) % tile_dim); + dims.set(d, std.math.divCeil(usize, dims.get(d), tile_dim) catch unreachable); + indices.set(d, indices.get(d) / tile_dim); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + + var offset: usize = 0; + for (dims.constSlice(), indices.constSlice()) |dim, coord| offset = offset * dim + coord; + return offset; +} + +test "LayoutMetadata byte offsets match physical layouts" { + const unit_tiles: [pjrt.DefaultMemoryLayout.MAX_TILE_DIMS]i64 = @splat(1); + const tile_ranks: [pjrt.DefaultMemoryLayout.MAX_NUM_TILES]usize = @splat(Shape.MAX_RANK); + const Case = struct { shape: Shape, layout: pjrt.MemoryLayout, block_size: usize }; + const cases = [_]Case{ + .{ .shape = .init(.{}, .i32), .layout = .{ .strides = .{ .byte_strides = &.{} } }, .block_size = 1 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, .block_size = 15 }, + .{ .shape = .init(.{ 3, 5 }, .f16), .layout = .{ .strides = .{ .byte_strides = &.{ 16, 2 } } }, .block_size = 5 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 4, 16 } } }, .block_size = 1 }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 15, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 4, 8 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile is larger than the values it tiles. + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 4, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile also splits the tile counts. + .{ + .shape = .init(.{ 4, 8 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 4, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2, 3, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile moves pairs of the innermost axis to the most minor dimension. + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 2, 1 }, .tile_dims = &.{ 2, 4, 2, 2, 1 }, .tile_dims_sizes = &.{ 3, 2 } } }, + .block_size = 2, + }, + // The tile has more dimensions than the shape. + .{ + .shape = .init(.{5}, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} } }, + .block_size = 4, + }, + .{ + .shape = .init(.{ 1, 1, 1, 1, 1, 1, 2, 3 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 7, 6, 5, 4, 3, 2, 1, 0 }, .tile_dims = &unit_tiles, .tile_dims_sizes = &tile_ranks } }, + .block_size = 3, + }, + }; + for (cases) |case| { + const metadata: LayoutMetadata = .init(case.shape, case.shape, case.layout); + try std.testing.expectEqual(case.block_size, metadata.block_size); + for (0..case.shape.count()) |local| { + const index = Coordinates.unflatten(case.shape, local); + const expected = referenceByteOffset(case.shape, case.layout, index.axes[0..case.shape.rank()]); + const actual = metadata.byteOffset(local, index); + try std.testing.expectEqual(expected, actual); + // Elements after the first of a block follow the previous one in memory. + if ((local % metadata.alignment_size) % metadata.block_size != 0) { + try std.testing.expectEqual(metadata.byteOffset(local - 1, Coordinates.unflatten(case.shape, local - 1)) + metadata.element_size, actual); + } + } + } +} + +test "tiled layout offsets follow XLA" { + const Tiled = pjrt.MemoryLayout.Tiled; + // Figure 1 of https://openxla.org/xla/tiled_layout: F32[3,5]{1,0:T(2,2)}. + const tiles: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(17, tiledElementOffset(.init(.{ 3, 5 }, .f32), tiles, &.{ 2, 3 })); + const shape_ = Shape.init(.{ 3, 4 }, .i32); + try std.testing.expectEqual(0, tiledElementOffset(shape_, tiles, &.{ 0, 0 })); + try std.testing.expectEqual(1, tiledElementOffset(shape_, tiles, &.{ 0, 1 })); + try std.testing.expectEqual(2, tiledElementOffset(shape_, tiles, &.{ 1, 0 })); + try std.testing.expectEqual(3, tiledElementOffset(shape_, tiles, &.{ 1, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(shape_, tiles, &.{ 0, 2 })); + try std.testing.expectEqual(8, tiledElementOffset(shape_, tiles, &.{ 2, 0 })); + + // Figure 2: 4x8 tiled by (2,4)(2,1). The nested tile pairs rows within each + // tile, as the (8,128)(2,1) tiling of bf16 on TPU does. + const nested: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } }; + const figure2 = [4][8]usize{ + .{ 0, 2, 4, 6, 8, 10, 12, 14 }, + .{ 1, 3, 5, 7, 9, 11, 13, 15 }, + .{ 16, 18, 20, 22, 24, 26, 28, 30 }, + .{ 17, 19, 21, 23, 25, 27, 29, 31 }, + }; + for (figure2, 0..) |offsets, row| { + for (offsets, 0..) |offset, col| try std.testing.expectEqual(offset, tiledElementOffset(.init(.{ 4, 8 }, .bf16), nested, &.{ row, col })); + } + + // A tile lists sizes for the most minor physical dimensions, major to minor. + // Like {2,1,0:T(8,128)} on TPU, this tiles rows by 2 and columns by 4: + // offset = a*32 + (b/2)*16 + (c/4)*8 + (b%2)*4 + c%4. + const rank3: Shape = .init(.{ 2, 3, 5 }, .f32); + const partial: Tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(4, tiledElementOffset(rank3, partial, &.{ 0, 1, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(rank3, partial, &.{ 0, 0, 4 })); + try std.testing.expectEqual(12, tiledElementOffset(rank3, partial, &.{ 0, 1, 4 })); + try std.testing.expectEqual(16, tiledElementOffset(rank3, partial, &.{ 0, 2, 0 })); + try std.testing.expectEqual(51, tiledElementOffset(rank3, partial, &.{ 1, 2, 3 })); + + // Tiles follow the physical order: offset = (c/2)*8 + (r/2)*4 + (c%2)*2 + r%2. + const transposed: Tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 1, 0 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 2 })); + try std.testing.expectEqual(14, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 3 })); + + // A tile with more dimensions than the shape pads it with leading unit + // dimensions: offset = (c/4)*8 + c%4. + const wide: Tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(3, tiledElementOffset(.init(.{5}, .f32), wide, &.{3})); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{5}, .f32), wide, &.{4})); + + // A later tile can also split the previous tile counts: + // offset = r*8 + (c%4)*2 + (c/4)%2. + const cross_tile: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 4 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 1 })); + try std.testing.expectEqual(11, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 1, 5 })); + try std.testing.expectEqual(31, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 3, 7 })); +} + +test "HostAccessible.HostPinnedAllocations keeps each region's replicas together as allocations are added" { + var allocations: HostAccessible.HostPinnedAllocations = .empty; + defer allocations.deinit(std.testing.allocator); + try allocations.entries.ensureTotalCapacity(std.testing.allocator, 5); + var storage: [5]u8 = undefined; + const base: [*]u8 = &storage; + // A vector of 12 split into shards of 4, holding regions A, B, A, C, B: + // replicas arrive interleaved with other regions. + const shard_shape: Shape = .init(.{4}, .u8); + for ([_]usize{ 0, 4, 0, 8, 4 }, 0..) |row, i| { + allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ @as([Shape.MAX_RANK - 1]usize, @splat(0)) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); + } + + try std.testing.expectEqualSlices([*]u8, &.{ base, base + 2, base + 1, base + 4, base + 3 }, allocations.entries.items(.ptr)); + for (allocations.regions.constSlice(), [_][2]u8{ .{ 0, 2 }, .{ 2, 2 }, .{ 4, 1 } }) |region, expected| { + try std.testing.expectEqual(expected, [2]u8{ region.start, region.len }); + } +} + +test "HostAccessible.format prints the elements in row-major order" { + const shape_ = Shape.init(.{ 2, 2 }, .i32); + const column_major: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + var storage = [_]i32{ 0, 2, 1, 3 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, column_major, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + var buffer: [128]u8 = undefined; + const printed = try std.fmt.bufPrint(&buffer, "{f}", .{pinned}); + try std.testing.expectStringEndsWith(printed, ") { 0 1 2 3 }"); +} + +test "HostAccessible.View borrows the buffer's pointers through slicing" { + const shape_ = Shape.init(.{4}, .i32); + const layout: pjrt.MemoryLayout = .{ .strides = .{ .byte_strides = &.{4} } }; + var storage = [_]i32{ 0, 1, 2, 3 }; + var other = [_]i32{ 10, 11, 12, 13 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const nested = pinned.view(i32).slice(1, null).slice(0, 2); + + // Slices read the pointers in place, which lets `await` refresh them + // without copying them into every view. + try std.testing.expectEqual(@as(*const HostAccessible.Mapping, pinned.mapping), nested.mapping); + pinned.mapping.allocations.entries.items(.ptr)[0] = @ptrCast(&other); + nested.fill(99); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3 }, &storage); + try std.testing.expectEqualSlices(i32, &.{ 10, 99, 99, 13 }, &other); +} + +test "HostAccessible.View blocks skip tile padding" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(2, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 0, .end = 2 }, values.blockBounds(1)); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 4, .end = 5 }, values.blockBounds(4)); + var iterator = values.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset..) |*value, index| value.* = @intCast(index + 1); + } + } + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 6, 7, 3, 4, 8, 9, 5, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); + values.slice(4, 3).fill(90); + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 90, 90, 3, 4, 8, 9, 90, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View nested tiles" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2, 2, 1 }, + .tile_dims_sizes = &.{ 2, 2 }, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + // Pairs of rows are interleaved within each tile, so every block is one element. + try std.testing.expectEqual(1, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }); + try std.testing.expectEqualSlices(i32, &.{ + 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, -1, -1, + 11, -1, 12, -1, 13, -1, 14, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View dense, transposed, strided and partial-rank tiled layouts" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + const Case = struct { layout: pjrt.MemoryLayout, block_size: usize, expected: []const i32 }; + const cases = [_]Case{ + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + .expected = &.{ 1, 4, 2, 5, 3, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, -1, 4, 5, 6 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 4, 12 } } }, + .block_size = 1, + .expected = &.{ 1, 4, -1, 2, 5, -1, 3, 6 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{2}, .tile_dims_sizes = &.{1} } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, 4, 5, 6, -1 }, + }, + }; + for (cases) |case| { + var storage: [8]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, case.layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6 }); + try std.testing.expectEqualSlices(i32, case.expected, &storage); + } +} + +test "HostAccessible.View row-major layouts return one block across rows and replicas" { + const shape_: Shape = .init(.{ 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + }; + for (layouts) |layout| { + var storage: [2][15]i32 = @splat(@splat(0)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + for ([_]HostAccessible.View(i32){ values, values.slice(1, null).slice(1, 10) }) |selected| { + var forward = selected.blocks(); + var reverse = selected.reverseBlocks(); + for ([_]HostAccessible.View(i32).Block{ forward.next().?, reverse.next().? }) |block| { + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(selected.len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[selected.start..].ptr, items.ptr); + try std.testing.expectEqual(selected.len, items.len); + } + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); + } + } +} + +test "HostAccessible.View dense shards stop at gaps in global row-major order" { + const Case = struct { + shape: Shape, + shard_shape: Shape, + block_size: usize, + origins: [2]usize, + expected: [2][]const i32, + }; + const cases = [_]Case{ + .{ + .shape = .init(.{ 2, 6 }, .i32), + .shard_shape = .init(.{ 2, 3 }, .i32), + .block_size = 3, + .origins = .{ 0, 3 }, + .expected = .{ &.{ 0, 1, 2, 6, 7, 8 }, &.{ 3, 4, 5, 9, 10, 11 } }, + }, + .{ + .shape = .init(.{ 2, 6, 2 }, .i32), + .shard_shape = .init(.{ 2, 3, 2 }, .i32), + .block_size = 6, + .origins = .{ 0, 6 }, + .expected = .{ + &.{ 0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17 }, + &.{ 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23 }, + }, + }, + }; + for (cases) |case| { + const strides = case.shard_shape.computeByteStrides(); + var storage: [2][12]i32 = @splat(@splat(-1)); + var ptrs: [2][*]u8 = undefined; + for (&ptrs, &storage) |*ptr, *data| ptr.* = @ptrCast(data); + const pinned = try HostAccessible.initForTests(std.testing.allocator, case.shape, case.shard_shape, .{ .strides = .{ .byte_strides = strides.constSlice() } }, &case.origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const elements = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(case.block_size, elements.blockBounds(0).end); + var tail = elements.slice(case.block_size - 1, null).blocks(); + try std.testing.expectEqual(1, tail.next().?.len); + + elements.fillIota(0, 1); + for (storage, case.expected) |data, expected| { + try std.testing.expectEqualSlices(i32, expected, data[0..expected.len]); + } + var actual: [24]i32 = undefined; + elements.copyTo(actual[0..case.shape.count()]); + for (actual[0..case.shape.count()], 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(i)), value); + + // A partial range crosses both a global gap and a physical shard boundary. + const start = case.block_size - 1; + const values = [_]i32{ 90, 91, 92, 93 }; + elements.slice(start, values.len).copyFrom(&values); + for (storage, case.expected) |data, expected| { + for (data[0..expected.len], expected) |value, global| { + const offset: usize = @intCast(global); + try std.testing.expectEqual(if (offset >= start and offset < start + values.len) values[offset - start] else global, value); + } + } + var copied: [4]i32 = undefined; + elements.slice(start, copied.len).copyTo(&copied); + try std.testing.expectEqualSlices(i32, &values, &copied); + } +} + +/// A 4x6 array split into 2x3 shards. Shards 4 to 7 are separate copies of shards 0 to 3. +const TestShards = struct { + storage: [8][8]i32 = @splat(@splat(-1)), + ptrs: [8][*]u8 = undefined, + origins: [8]usize = @splat(0), + pinned: ?HostAccessible = null, + + const shape_ = Shape.init(.{ 4, 6 }, .i32); + const shard_shape = Shape.init(.{ 2, 3 }, .i32); + const tiled: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + + fn deinit(self: *TestShards) void { + if (self.pinned) |pinned| pinned.deinit(std.testing.allocator); + } + + /// The view borrows `pinned`, which borrows the storage: the storage must + /// stay in place, and the next call replaces `pinned`. + fn view(self: *TestShards, layout: pjrt.MemoryLayout) !HostAccessible.View(i32) { + for (&self.ptrs, &self.origins, &self.storage, 0..) |*ptr, *origin, *data, i| { + ptr.* = @ptrCast(data); + const row = ((i % 4) / 2) * 2; + const column = (i % 2) * 3; + origin.* = row * 6 + column; + } + self.deinit(); + self.pinned = null; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, layout, &self.origins, &self.ptrs); + self.pinned = pinned; + return pinned.view(i32); + } + + /// With the tiled layout, every copy of each element holds `expected[global]` + /// and the tile padding is untouched. + fn expectValues(self: *const TestShards, expected: [24]i32) !void { + // Explicit offsets for the two rows of a 2x3 shard tiled by 2x2. + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + for (self.storage, self.origins) |data, origin| { + for (physical_offsets, 0..) |physical, local| { + const global = origin + (local / 3) * 6 + local % 3; + try std.testing.expectEqual(expected[global], data[physical]); + } + try std.testing.expectEqual(-1, data[5]); + try std.testing.expectEqual(-1, data[7]); + } + } +}; + +test "HostAccessible.View blocks visit partial global ranges in every shard" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var visits: [24]u8 = @splat(0); + // Nested slicing must keep block offsets relative to the selected range. + var iterator = values.slice(1, null).slice(1, 18).blocks(); + while (iterator.next()) |block| { + try std.testing.expect(block.offset + block.len <= 18); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset + 2..) |*value, global| { + visits[global] += 1; + value.* = @intCast(global); + } + } + } + var expected: [24]i32 = @splat(-1); + for (visits, &expected, 0..) |count, *value, global| { + const selected = global >= 2 and global < 20; + try std.testing.expectEqual(@as(u8, if (selected) 2 else 0), count); + if (selected) value.* = @intCast(global); + } + try fixture.expectValues(expected); + // Starting after the first row partition must skip those shards entirely. + values.set(17, 99); + try std.testing.expectEqual(99, fixture.storage[3][4]); + try std.testing.expectEqual(99, fixture.storage[7][4]); + + const Case = struct { step: i32, expected: [18]i32 }; + const cases = [_]Case{ + .{ .step = 3, .expected = .{ 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 37, 40, 43, 46, 49, 52, 55, 58 } }, + .{ .step = -2, .expected = .{ 7, 5, 3, 1, -1, -3, -5, -7, -9, -11, -13, -15, -17, -19, -21, -23, -25, -27 } }, + .{ .step = 0, .expected = @splat(7) }, + }; + for (cases) |case| { + values.slice(1, null).slice(1, 18).fillIota(7, case.step); + expected[2..20].* = case.expected; + try fixture.expectValues(expected); + } +} + +test "HostAccessible.View blocks merge shuffled shards and group replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + _ = try fixture.view(TestShards.tiled); + const order = [_]usize{ 3, 4, 1, 6, 2, 0, 7, 5 }; + var ptrs: [8][*]u8 = undefined; + var origins: [8]usize = undefined; + for (order, &ptrs, &origins) |index, *ptr, *origin| { + ptr.* = fixture.ptrs[index]; + origin.* = fixture.origins[index]; + } + const pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shape_, TestShards.shard_shape, TestShards.tiled, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + const selected = values.slice(1, null).slice(1, 18); + const offsets = [_]usize{ 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16 }; + const lengths = [_]usize{ 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 }; + const replicas = [_][2]usize{ .{ 4, 0 }, .{ 1, 5 }, .{ 6, 2 }, .{ 3, 7 } }; + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + var iterator = selected.blocks(); + for (offsets, lengths) |offset, len| { + const block = iterator.next().?; + try std.testing.expectEqual(offset, block.offset); + try std.testing.expectEqual(len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + const block_start = offset + 2; + const region = block_start / 12 * 2 + block_start % 6 / 3; + const physical = physical_offsets[block_start / 6 % 2 * 3 + block_start % 3]; + for (0..block.replicas.len, replicas[region]) |i, storage_index| { + const items = block.replica(i); + try std.testing.expectEqual(fixture.storage[storage_index][physical..].ptr, items.ptr); + } + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, offset + 2..) |*value, global| value.* = @intCast(global); + } + } + try std.testing.expectEqual(null, iterator.next()); + var expected: [24]i32 = @splat(-1); + for (expected[2..20], 2..) |*value, global| value.* = @intCast(global); + try fixture.expectValues(expected); + try expectReverseBlocksMirrorForward(selected); +} + +test "HostAccessible.View blocks group the maximum number of replicas" { + var storage: [Platform.MAX_NUM_DEVICES][4]i32 = @splat(@splat(-1)); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + const origins: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (&ptrs, &storage) |*ptr, *items| ptr.* = @ptrCast(items); + const shape_: Shape = .init(.{4}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + var iterator = values.blocks(); + const block = iterator.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(4, block.len); + try std.testing.expectEqual(Platform.MAX_NUM_DEVICES, block.replicas.len); + try std.testing.expectEqual(null, iterator.next()); + // The descriptors remain valid after advancing the iterator. + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[0..].ptr, items.ptr); + @memset(items, 42); + try std.testing.expectEqualSlices(i32, &.{ 42, 42, 42, 42 }, replica); + } + try expectReverseBlocksMirrorForward(values); +} + +test "HostAccessible.View visits the maximum number of distinct shuffled regions" { + var storage: [Platform.MAX_NUM_DEVICES]i32 = @splat(-1); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + var origins: [Platform.MAX_NUM_DEVICES]usize = undefined; + for (&ptrs, &origins, 0..) |*ptr, *origin, shard| { + const index = storage.len - shard - 1; + ptr.* = @ptrCast(&storage[index]); + origin.* = index; + } + const shape_: Shape = .init(.{Platform.MAX_NUM_DEVICES}, .i32); + const shard_shape: Shape = .init(.{1}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var forward = values.blocks(); + var reverse = values.reverseBlocks(); + for (storage, 0..) |value, index| { + try std.testing.expectEqual(@as(i32, @intCast(index)), value); + const block = forward.next().?; + try std.testing.expectEqual(index, block.offset); + try std.testing.expectEqual(1, block.len); + try std.testing.expectEqual(1, block.replicas.len); + try std.testing.expectEqual(storage[index..].ptr, block.firstReplica().ptr); + const reverse_block = reverse.next().?; + const reverse_index = storage.len - index - 1; + try std.testing.expectEqual(reverse_index, reverse_block.offset); + try std.testing.expectEqual(storage[reverse_index..].ptr, reverse_block.firstReplica().ptr); + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View multidimensional slices preserve coordinates across carries" { + const shape_: Shape = .init(.{ 2, 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 60, 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + }; + for (layouts) |layout| { + var storage: [48]i32 = @splat(-1); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var expected: [48]i32 = @splat(-1); + for (0..shape_.count()) |local| { + const index = Coordinates.unflatten(shape_, local); + const physical = referenceByteOffset(shape_, layout, index.axes[0..shape_.rank()]) / @sizeOf(i32); + expected[physical] = @intCast(local); + try std.testing.expectEqual(@as(i32, @intCast(local)), values.get(local)); + } + try std.testing.expectEqualSlices(i32, &expected, &storage); + for (0..shape_.count() + 1) |start| { + for (0..shape_.count() - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } +} + +test "HostAccessible.View reverse blocks mirror forward blocks" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const layouts = [_]pjrt.MemoryLayout{ + TestShards.tiled, + .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .{ .strides = .{ .byte_strides = &.{ 4, 8 } } }, + }; + for (layouts) |layout| { + const values = try fixture.view(layout); + for (0..values.len + 1) |start| { + for (0..values.len - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } + // Dense blocks can span multiple rows, and scalar blocks have no row axis. + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shard_shape, TestShards.shard_shape, layouts[1], fixture.origins[0..1], fixture.ptrs[0..1]); + defer dense_pinned.deinit(std.testing.allocator); + const dense = dense_pinned.view(i32); + try expectReverseBlocksMirrorForward(dense.slice(1, 4)); + const scalar_shape: Shape = .init(.{}, .i32); + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, scalar_shape, scalar_shape, .{ .strides = .{ .byte_strides = &.{} } }, fixture.origins[0..1], fixture.ptrs[0..1]); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar = scalar_pinned.view(i32); + try expectReverseBlocksMirrorForward(scalar); +} + +fn expectReverseBlocksMirrorForward(values: HostAccessible.View(i32)) !void { + var expected: [48]HostAccessible.View(i32).Block = undefined; + var len: usize = 0; + var offset: usize = 0; + var forward = values.blocks(); + while (forward.next()) |block| : (len += 1) { + try std.testing.expectEqual(offset, block.offset); + expected[len] = block; + offset += block.len; + } + try std.testing.expectEqual(values.len, offset); + var reverse = values.reverseBlocks(); + while (len > 0) { + len -= 1; + const block = reverse.next().?; + try std.testing.expectEqual(expected[len].offset, block.offset); + try std.testing.expectEqual(expected[len].len, block.len); + try std.testing.expectEqual(expected[len].byte_offset, block.byte_offset); + try std.testing.expectEqualSlices([*]u8, expected[len].replicas, block.replicas); + } + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View scalar, empty ranges and rank-one padding" { + var scalar: i32 = 0; + const scalar_ptrs = [_][*]u8{@ptrCast(&scalar)}; + const shape_ = Shape.scalar(.i32); + const scalar_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{}, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, scalar_layout, &.{0}, &scalar_ptrs); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar_view = scalar_pinned.view(i32); + scalar_view.copyFrom(&.{42}); + try std.testing.expectEqual(42, scalar); + var past_end = scalar_view.slice(1, null).blocks(); + try std.testing.expectEqual(null, past_end.next()); + + var storage: [8]bool = @splat(false); + const vector_ptrs = [_][*]u8{@ptrCast(&storage)}; + const vector_shape = Shape.init(.{5}, .bool); + const vector_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{4}, .tile_dims_sizes = &.{1} } }; + const vector_pinned = try HostAccessible.initForTests(std.testing.allocator, vector_shape, vector_shape, vector_layout, &.{0}, &vector_ptrs); + defer vector_pinned.deinit(std.testing.allocator); + const vector = vector_pinned.view(bool); + vector.fill(true); + try std.testing.expectEqualSlices(bool, &.{ true, true, true, true, true, false, false, false }, &storage); + + const empty_shape = Shape.init(.{0}, .bool); + const empty_pinned = try HostAccessible.initForTests(std.testing.allocator, empty_shape, empty_shape, vector_layout, &.{0}, &vector_ptrs); + defer empty_pinned.deinit(std.testing.allocator); + const empty_view = empty_pinned.view(bool); + var empty_blocks = empty_view.blocks(); + try std.testing.expectEqual(null, empty_blocks.next()); + + // The empty view has no storage or metadata to read. + const none: HostAccessible.View(u32) = .empty; + none.fill(0); + none.copyTo(&.{}); + var none_blocks = none.reverseBlocks(); + try std.testing.expectEqual(null, none_blocks.next()); +} + +test "HostAccessible devices holding a shard read one allocation" { + const platform = testing.env(); + const tp = testing.tpMesh(); + + const shape_ = Shape.init(.{ .b = @as(i64, @intCast(platform.devices.len)), .d = 5 }, .i32); + try expectDevicesReadShards(platform, shape_); + try expectDevicesReadShards(platform, shape_.withPartitioning(tp, .{ .b = .model })); +} + +test "HostAccessible devices holding a shard read one allocation in data-parallel groups" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Two data-parallel groups of two tensor-parallel devices. + const platform = Platform.auto(allocator, io, .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init( + "data_parallel", + &platform.physical_mesh, + .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .parseBindings(.{ .batch = .link_x, .model = .link_y }), + ); + try expectDevicesReadShards(platform, Shape.init(.{ .b = 4, .d = 5 }, .i32).withPartitioning(&mesh, .{ .b = .batch })); +} + +/// Rows of `.d` elements written once through a view reach every device +/// holding them, and devices holding the same shard share its allocation +/// where the platform allows it. +fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { + if (!builtin.is_test) @compileError("expectDevicesReadShards is only available in tests"); + + const allocator = std.testing.allocator; + const io = std.testing.io; + const api = platform.pjrt_api; + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + const placement: Sharding.Placement = try Sharding.Placement.init(&shape_); + const expected_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => shape_.count() / placement.localShape(shape_).count(), + else => platform.devices.len, + }; + try std.testing.expectEqual(expected_allocations, pinned.mapping.allocations.entries.len); + + pinned.view(i32).fillIota(0, 1); + const row_size: usize = @intCast(shape_.dim(.d)); + const values = try allocator.alloc(i32, placement.localShape(shape_).count()); + defer allocator.free(values); + for (pinned.buffer._shards.constSlice(), 0..) |shard, device_index| { + @memset(values, -1); + if (try shard.toHostBuffer(api, std.mem.sliceAsBytes(values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + const first_row = Coordinates.ofShard(&placement, device_index, shape_).axes[0]; + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(first_row * row_size + i)), value); + } +} + +test "HostAccessible executions read shared inputs on every device" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const api = platform.pjrt_api; + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + for (0..2) |iteration| { + pinned.view(i32).fillIota(@intCast(iteration * 100), 1); + var output: Buffer = undefined; + runner.run(io, .{pinned.buffer}, .{&output}, .{}); + defer output.deinit(); + try output.await(io); + for (output._shards.constSlice()) |shard| { + var values: [15]i32 = @splat(-1); + if (try shard.toHostBuffer(api, std.mem.asBytes(&values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + } + } +} + +test "HostAccessible.View.constSlice borrows contiguous buffers and copies the others" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + var dense: [2][6]i32 = .{ .{ 0, 1, 2, 3, 4, 5 }, @splat(-1) }; + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const borrowed = dense_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, borrowed.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, borrowed); + + var column_major = [_]i32{ 0, 3, 1, 4, 2, 5 }; + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied = transposed_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(@as([*]const i32, &scratch), copied.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, copied); + + const none: HostAccessible.View(i32) = .empty; + try std.testing.expectEqual(0, none.constSlice(&.{}).len); +} + +test "HostAccessible.View.edit writes contiguous buffers in place and commits every replica" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + // In place: the first replica takes the writes, and `commit` copies it to the others. + var dense: [2][6]i32 = @splat(@splat(0)); + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const dense_edit = dense_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, dense_edit.items.ptr); + @memcpy(dense_edit.items, &[_]i32{ 1, 2, 3, 4 }); + dense_edit.commit(); + for (dense) |replica| try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 0 }, &replica); + + // Through the scratch: the buffer only changes on `commit`. + var column_major: [6]i32 = @splat(0); + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied_edit = transposed_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(@as([*]i32, &scratch), copied_edit.items.ptr); + @memcpy(copied_edit.items, &[_]i32{ 1, 2, 3, 4 }); + try std.testing.expectEqualSlices(i32, &.{ 0, 0, 0, 0, 0, 0 }, &column_major); + copied_edit.commit(); + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 4, 2, 0 }, &column_major); + + const none: HostAccessible.View(i32) = .empty; + none.edit(&.{}).commit(); +} + +test "HostAccessible.View.indexOfScalar and indexOfAny search blocks in row-major order" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fillIota(0, 1); + values.set(20, 7); + // Tiles split the view into blocks, and shards into regions. + try std.testing.expectEqual(7, values.indexOfScalar(7)); + try std.testing.expectEqual(12, values.slice(8, null).indexOfScalar(7)); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfScalar(7)); + try std.testing.expectEqual(null, values.indexOfScalar(-1)); + try std.testing.expectEqual(null, HostAccessible.View(i32).empty.indexOfScalar(0)); + + try std.testing.expectEqual(5, values.indexOfAny(&.{ 7, 5 })); + try std.testing.expectEqual(12, values.slice(8, null).indexOfAny(&.{ -1, 7 })); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfAny(&.{ 7, -1 })); + try std.testing.expectEqual(null, values.indexOfAny(&.{})); +} + +test "HostAccessible.View indexes replicas of a dense array directly" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var storage: [2][6]i32 = @splat(@splat(-1)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + try std.testing.expect(pinned.mapping.contiguous); + const values = pinned.view(i32); + for (0..6) |i| values.set(i, @intCast(10 + i)); + values.slice(4, null).set(1, 99); + for (storage) |replica| try std.testing.expectEqualSlices(i32, &.{ 10, 11, 12, 13, 14, 99 }, &replica); + try std.testing.expectEqual(99, values.get(5)); + try std.testing.expectEqual(13, values.slice(2, 3).get(1)); + + inline for (.{ false, true }) |reverse| { + const nested = values.slice(1, 4); + var blocks = if (reverse) nested.reverseBlocks() else nested.blocks(); + const block = blocks.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqualSlices(i32, &.{ 11, 12, 13, 14 }, items); + try std.testing.expectEqual(replica[1..].ptr, items.ptr); + } + try std.testing.expectEqual(null, blocks.next()); + } +} + +test "HostAccessible.View set and get address single elements in every replica" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var expected: [24]i32 = undefined; + var i: usize = expected.len; + while (i > 0) { + i -= 1; + expected[i] = @intCast(100 + i); + values.set(i, expected[i]); + } + try fixture.expectValues(expected); + for (expected, 0..) |value, index| try std.testing.expectEqual(value, values.get(index)); + + // Indices of a nested view are relative to its start. + const nested = values.slice(7, 10); + nested.set(3, -5); + expected[10] = -5; + try fixture.expectValues(expected); + try std.testing.expectEqual(-5, nested.get(3)); +} + +test "HostAccessible.View copies across layouts, shards and replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fill(0); + values.slice(2, 20).fillIota(10, 1); + values.set(17, 99); + const expected = [_]i32{ 0, 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 99, 26, 27, 28, 29, 0, 0 }; + var actual: [24]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); + for (expected, 0..) |value, i| try std.testing.expectEqual(value, values.get(i)); + try fixture.expectValues(expected); + + // Blocks expose every replica, respect physical boundaries, and stop at + // the end of a nested view even when the physical run continues. + const nested = values.slice(1, null).slice(2, 4); + var nested_blocks = nested.blocks(); + const first = nested_blocks.next().?; + try std.testing.expectEqual(0, first.offset); + try std.testing.expectEqual(2, first.replicas.len); + for (0..first.replicas.len, [_]usize{ 1, 5 }) |i, storage_index| { + const items = first.replica(i); + try std.testing.expectEqualSlices(i32, expected[3..5], items); + try std.testing.expectEqual(fixture.storage[storage_index][0..].ptr, items.ptr); + } + const middle = nested_blocks.next().?; + try std.testing.expectEqual(2, middle.offset); + for (0..middle.replicas.len) |i| try std.testing.expectEqualSlices(i32, expected[5..6], middle.replica(i)); + const last = nested_blocks.next().?; + try std.testing.expectEqual(3, last.offset); + for (0..last.replicas.len, [_]usize{ 0, 4 }) |i, storage_index| { + const items = last.replica(i); + try std.testing.expectEqualSlices(i32, expected[6..7], items); + try std.testing.expectEqual(fixture.storage[storage_index][2..].ptr, items.ptr); + } + try std.testing.expectEqual(null, nested_blocks.next()); + + // Copy between differently shaped/tiled ranges, including a nonzero source + // and destination start, and verify every destination replica physically. + const dst_shape = Shape.init(.{ 3, 7 }, .i32); + const dst_layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 0, 1 }, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + } }; + var dst_storage: [2][21]i32 = @splat(@splat(-1)); + const dst_ptrs = [_][*]u8{ @ptrCast(&dst_storage[0]), @ptrCast(&dst_storage[1]) }; + const dst_pinned = try HostAccessible.initForTests(std.testing.allocator, dst_shape, dst_shape, dst_layout, &.{ 0, 0 }, &dst_ptrs); + defer dst_pinned.deinit(std.testing.allocator); + const dst = dst_pinned.view(i32); + dst.slice(1, 19).copyFromView(values.slice(1, 22).slice(1, 19)); + for (dst_storage) |data| { + for (0..21) |i| { + const physical = (i % 7) * 3 + i / 7; + try std.testing.expectEqual(if (i > 0 and i < 20) expected[i + 1] else -1, data[physical]); + } + } + var empty: [0]i32 = .{}; + values.slice(values.len, 0).copyTo(&empty); + values.slice(0, 0).copyFromView(dst.slice(0, 0)); + values.slice(values.len, null).fill(-1); + values.slice(values.len, null).fillIota(-1, 1); + values.slice(values.len, null).copyFrom(&empty); + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); +} + +test "HostAccessible await after executable donation permits the next host update" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + const host_input = input.onMemory(.host_pinned); + return host_input.toMemory(.device).addConstant(1).toMemory(.host_pinned).reuseBuffer(host_input); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + try std.testing.expectEqual(pinned.buffer._shards.len, pinned.mapping.allocations.entries.len); + const first_ptr = pinned.mapping.allocations.entries.items(.ptr)[0]; + pinned.view(i32).fillIota(0, 1); + + // Reuse the same runner and output destination as the model execution paths. + // Only await waits for the asynchronous executable before host access. + for (0..3) |iteration| { + runner.run(io, .{pinned.buffer}, .{&pinned.buffer}, .{}); + try pinned.await(io); + + // Reacquire from the output handles, even when donation reuses memory. + const values = pinned.view(i32); + var actual: [15]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqual(first_ptr, pinned.mapping.allocations.entries.items(.ptr)[0]); + for (actual, 0..) |value, i| { + const expected: i32 = if (iteration == 0 or i < 3 or i >= 12) + @intCast(i + iteration + 1) + else + @intCast(iteration * 100 + i - 3 + 1); + try std.testing.expectEqual(expected, value); + } + + // Also check the executable's logical output through PJRT, independently + // of the host-visible layout used above. + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + values.slice(3, 9).fillIota(@intCast((iteration + 1) * 100), 1); + } +} + +test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + // Without reuseBuffer, execution writes a new host allocation. + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1).toMemory(.host_pinned); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + pinned.view(i32).fillIota(0, 1); + + for (0..2) |iteration| { + // The input isn't donated, so release it once execution replaced it. + var input = pinned.buffer; + defer input.deinit(); + const previous = pinned.mapping.allocations.entries.items(.ptr)[0]; + runner.run(io, .{input}, .{&pinned.buffer}, .{}); + try pinned.await(io); + const values = pinned.view(i32); + try std.testing.expect(pinned.mapping.allocations.entries.items(.ptr)[0] != previous); + + var actual: [15]i32 = undefined; + values.copyTo(&actual); + for (actual, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + // Host writes now land in the output, which the next execution reads. + values.fillIota(@intCast((iteration + 1) * 100), 1); + } +} diff --git a/zml/constants.zig b/zml/constants.zig index ad8ff445..39586fa4 100644 --- a/zml/constants.zig +++ b/zml/constants.zig @@ -13,3 +13,6 @@ pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = b: { pub fn minorToMajor(rank: u8) []const i64 { return MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - rank ..]; } + +pub const ones_i64: [MAX_RANK]i64 = @splat(1); +pub const zeroes_i64: [MAX_RANK]i64 = @splat(0); diff --git a/zml/dtype.zig b/zml/dtype.zig index 8ec73c3d..f5cd9da8 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,5 +1,7 @@ const std = @import("std"); +const stdx = @import("stdx"); + const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); @@ -7,8 +9,8 @@ const C128 = std.math.Complex(f64); pub const DataType = enum(u8) { bool, - // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. f4e2m1, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. f8e3m4, f8e4m3, f8e4m3b11fnuz, @@ -136,9 +138,23 @@ pub const DataType = enum(u8) { return @FieldType(Value, @tagName(dtype)); } + pub fn toPackedZigType(comptime dtype: DataType) type { + return switch (dtype) { + .bool => bool, + .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), + .f4e2m1 => floats.Float4E2M1.Packed, + inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), + else => { + const T = @FieldType(Value, @tagName(dtype)); + if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); + return T; + }, + }; + } + pub fn isSignedInt(dtype: DataType) bool { return switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => true, + .i2, .i4, .i8, .i16, .i32, .i64 => true, else => false, }; } @@ -151,6 +167,9 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { + .bool => 8, // In PJRT/stablehlo bool always occupy a full byte + .c64 => 64, + .c128 => 128, inline else => |tag| @bitSizeOf(tag.toZigType()), }; } @@ -344,7 +363,7 @@ pub const DataType = enum(u8) { .bool => return self.bool, .float => switch (self) { inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |v| return @floatCast(v.toF32()), else => {}, }, .int => switch (self) { @@ -355,6 +374,13 @@ pub const DataType = enum(u8) { } std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); } + + test as { + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f4e2m1, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e3m4, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e4m3, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e8m0, 1).as(f32)); + } }; }; diff --git a/zml/exe.zig b/zml/exe.zig index 67493887..32c0ac48 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -1,12 +1,18 @@ const std = @import("std"); +const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; +const Compiler = @import("Compiler.zig"); +const mem = @import("mem.zig"); const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; pub const Exe = struct { platform: *const Platform, @@ -17,7 +23,12 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - num_devices: u8, + /// Inputs whose buffers are donated to an output (`reuseBuffer`). + /// Runners destroy their PJRT handles after the raw call returns. + donated_input_indices: []const u32, + + num_devices: u32, + num_partitions: u32, arena: std.heap.ArenaAllocator, @@ -25,9 +36,11 @@ pub const Exe = struct { allocator: std.mem.Allocator, platform: *const Platform, exe: *pjrt.LoadedExecutable, + num_devices: u32, + num_partitions: u32, input_shapes: []const Shape, output_shapes: []const Shape, - num_devices: u8, + input_aliasing: []const ?u32, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); @@ -35,17 +48,33 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + var donated_count: usize = 0; + for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); + + const donated_input_indices = try arena.allocator().alloc(u32, donated_count); + + var donated_index: usize = 0; + for (input_aliasing, 0..) |aliasing, input_index| { + if (aliasing == null) continue; + donated_input_indices[donated_index] = @intCast(input_index); + donated_index += 1; + } + return .{ .platform = platform, .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, + .donated_input_indices = donated_input_indices, .num_devices = num_devices, + .num_partitions = num_partitions, .arena = arena, }; } pub fn deinit(self: *const Exe) void { + if (self.context) |context| context.deinit(self.platform.pjrt_api); + self.exe.deinit(self.platform.pjrt_api); self.arena.deinit(); } @@ -54,7 +83,7 @@ pub const Exe = struct { } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); + return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); } pub const FlatBuffers = struct { @@ -92,7 +121,7 @@ pub const Exe = struct { expected_shapes: []const Shape, baked_count: usize = 0, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -122,9 +151,24 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { - stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - for (0..context_.self.flat_buffers.num_devices) |device_index| { - context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + stdx.debug.assert( + context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), + "Expected argument {} to have shape {f}, got {f}", + .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }, + ); + + const expected = context_.self.flat_buffers.num_devices; + const shard_count = buffer._shards.len; + + stdx.debug.assert( + shard_count == expected, + "Argument {} has {d} shards but executable expects {d}", + .{ context_.current_index, shard_count, expected }, + ); + + for (0..expected) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = + buffer._shards.get(device_index); } context_.current_index += 1; @@ -163,7 +207,7 @@ pub const Exe = struct { expected_shapes: []const Shape, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: *const Platform) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -191,7 +235,7 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = 0 }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *Buffer) void { - var shards: Buffer.Shards = .{}; + var shards: Buffer.Shards = .empty; for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } @@ -204,62 +248,457 @@ pub const Exe = struct { pub fn fill(self: *Results, v: anytype) void { const LocalContext = struct { - self: *Results, + results: *Results, current_index: usize = 0, }; - var context: LocalContext = .{ .self = self, .current_index = 0 }; + var context: LocalContext = .{ .results = self, .current_index = 0 }; meta.visit(struct { - fn cb(context_: *LocalContext, buffer: *Buffer) void { - //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); - var shards: Buffer.Shards = .{}; - for (0..context_.self.flat_buffers.num_devices) |device_index| { - shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + fn cb(ctx: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(ctx.results.expected_shapes[ctx.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ ctx.current_index, ctx.results.expected_shapes[ctx.current_index], buffer.shape() }); + var shards: Buffer.Shards = .empty; + for (0..ctx.results.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); - context_.current_index += 1; + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); + ctx.current_index += 1; } }.cb, &context, &v); } }; - pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); - var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; - const sharding = self.platform.sharding(); - const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; + /// Executes a single-output executable and waits for its result. + /// The caller owns the returned buffer. Donated inputs are consumed. + pub fn eval(self: *const Exe, allocator: std.mem.Allocator, io: std.Io, input_values: anytype) !Buffer { + stdx.debug.assert(self.output_shapes.len == 1, "eval expects one output, got {d}; use a runner for multiple outputs", .{self.output_shapes.len}); + var result: Buffer = undefined; + var r = try self.runner(allocator); + defer r.deinit(allocator); + r.run(io, input_values, .{&result}, .{ .wait = true }); + return result; + } + + pub fn runner(self: *const Exe, allocator: std.mem.Allocator) !Runner { + return .init(self, allocator); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. + pub const Runner = struct { + exe: *const Exe, + args: Arguments, + results: Results, + + pub fn init(exe: *const Exe, allocator: std.mem.Allocator) !Runner { + var arguments = try exe.args(allocator); + errdefer arguments.deinit(allocator); + var results_ = try exe.results(allocator); + errdefer results_.deinit(allocator); + return .{ + .exe = exe, + .args = arguments, + .results = results_, + }; + } + + pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + } + + /// Destroys donated input handles and fills the output destinations. + /// Use the donated buffer's address as its output destination to replace it safely. + /// Non-donated inputs remain owned by the caller. + pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { + self.args.set(input_values); + self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); + self.results.fill(output_refs); + } + }; + + pub const CallOpts = struct { + /// Waits for execution to complete before returning. + wait: bool = false, + }; + + fn deinitDonatedInputs(self: *const Exe, arguments: Arguments) void { + // Execution deletes donated inputs, but leaves their PJRT handles allocated. + // Arguments retain the old handles even when outputs replace the caller's buffers. + for (self.donated_input_indices) |input_index| { + for (arguments.flat_buffers.buffers[0..self.num_devices]) |device_buffers| { + device_buffers[input_index].deinit(self.platform.pjrt_api); + } + } + } + + /// Executes with optional waiting. The caller owns all input handles, including donated inputs. + /// Donated input contents cannot be used after the call; output buffers have their own handles. + /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + var span = tracer.span("zml.exe.call", .{ + .wait = opts.wait, + .arg_count = arguments.expected_shapes.len, + .result_count = results_.expected_shapes.len, + }); + defer span.end(); + + var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); + + const partition_events = events[0..@intCast(self.num_partitions)]; + const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { + .neuron => partition_events, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, + }; self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, .events = events_slice, - // this allows to tell a specific buffer shouldn't be donated, - // even if it has been marked as "can be donated" during compilation. - // TODO: expose it ? + // TODO: Set this flag for undonatable buffers. .non_donatable_input_indices = &.{}, .context = self.context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io.?) catch unreachable; + switch (self.platform.target) { + .neuron => { + for (events_slice.?) |e| { + if (e) |ev| { + if (opts.wait) { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + } + ev.deinit(self.platform.pjrt_api); + } } - } + }, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + ev.deinit(self.platform.pjrt_api); + } + } + }, } } - pub const CallOpts = struct { - wait: bool = false, - }; - - pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - return self.internalCall(io, arguments, results_, opts); - } - - pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { - return self.internalCall(null, arguments, results_, .{}); + pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { + const exe = try self.exe.executable(self.platform.pjrt_api); + return try exe.getCompiledMemoryStats(self.platform.pjrt_api); } }; + +/// A typed executable whose input and output buffer structures are derived from +/// the function used to compile it. +pub fn FnExe(comptime function_: anytype) type { + const function_info = switch (@typeInfo(@TypeOf(function_))) { + .@"fn" => |info| info, + else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), + }; + if (function_info.attrs.varargs or function_info.param_types.len != 1) { + @compileError("FnExe function must accept exactly one input struct"); + } + + const FunctionInput = typedFunctionInput(function_); + const FunctionOutput = typedFunctionOutput(function_); + + return struct { + raw: Exe, + + pub const Input = mem.Bufferized(FunctionInput); + pub const Output = typedOutputDestinations(mem.Bufferized(FunctionOutput)); + + const Self = @This(); + + pub fn init(raw: Exe) Self { + return .{ .raw = raw }; + } + + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + args: std.meta.ArgsTuple(@TypeOf(function_)), + ) Compiler.Error!Self { + return .{ .raw = try Compiler.Typed(function_).compile(allocator, io, platform, opts, args) }; + } + + pub fn deinit(self: *const Self) void { + self.raw.deinit(); + } + + pub fn runner(self: *const Self, allocator: std.mem.Allocator) !Runner(.{}) { + return .init(self, allocator, .{}); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. + pub fn Runner(comptime baked_fields: anytype) type { + const count = countBackedFields(Input, baked_fields); + + return struct { + exe: *const Exe, + args: Exe.Arguments, + results: Exe.Results, + + const RunnerSelf = @This(); + + pub const BakedInput = structFieldRange(Input, 0, count); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); + + pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { + var arguments = try exe.raw.args(allocator); + errdefer arguments.deinit(allocator); + var results = try exe.raw.results(allocator); + errdefer results.deinit(allocator); + arguments.bake(baked); + return .{ .exe = &exe.raw, .args = arguments, .results = results }; + } + + pub fn deinit(self: *RunnerSelf, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + } + + pub const Call = struct { + inputs: NonBakedInput, + outputs: Output, + }; + + /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. + pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { + self.args.set(call.inputs); + self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); + self.results.fill(call.outputs); + } + }; + } + }; +} + +fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize { + const baked_info = switch (@typeInfo(@TypeOf(baked_fields))) { + .@"struct" => |info| info, + else => @compileError("FnExe baked fields must be a tuple of enum literals"), + }; + if (!baked_info.is_tuple) { + @compileError("FnExe baked fields must be a tuple of enum literals"); + } + + const input_info = @typeInfo(Input).@"struct"; + if (baked_info.field_names.len > input_info.field_names.len) { + @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); + } + + inline for (baked_info.field_names, 0..) |tuple_field_name, i| { + const baked_field = @field(baked_fields, tuple_field_name); + if (@TypeOf(baked_field) != @EnumLiteral()) { + @compileError("FnExe baked fields must be enum literals"); + } + if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { + @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); + } + } + return baked_info.field_names.len; +} + +fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { + const info = @typeInfo(Struct).@"struct"; + return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); +} + +fn typedFunctionInput(comptime function_: anytype) type { + const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse + @compileError("FnExe function must have a concrete input type"); + validateFnExeStruct("input", Input); + return Input; +} + +fn typedFunctionOutput(comptime function_: anytype) type { + const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("FnExe function must return an output struct"); + validateFnExeStruct("output", Output); + return Output; +} + +fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { + const info = switch (@typeInfo(T)) { + .@"struct" => |info| info, + else => @compileError("FnExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), + }; + if (info.is_tuple) { + @compileError("FnExe " ++ role ++ " must use named fields"); + } +} + +fn typedOutputDestinations(comptime BufferizedOutput: type) type { + const output_info = @typeInfo(BufferizedOutput).@"struct"; + var field_types: [output_info.field_names.len]type = undefined; + var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { + T.* = *Field; + attrs.* = .{ .@"align" = @alignOf(*Field) }; + } + return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); +} + +test "FnExe derives baked and runtime calls from a function" { + const Inputs = struct { + weights: struct { + tensor: Tensor, + scale: f32, + }, + bias: Tensor, + tokens: Tensor, + token_count: usize, + }; + const Outputs = struct { + hidden: Tensor, + cache: struct { + key: Tensor, + length: usize, + }, + metadata: usize, + }; + const Functions = struct { + fn forward(inputs: Inputs) Outputs { + _ = inputs; + return undefined; + } + }; + + const Model = FnExe(Functions.forward); + try std.testing.expect(@hasField(Model.Input, "weights")); + try std.testing.expect(@hasField(Model.Input, "bias")); + try std.testing.expect(@hasField(Model.Input, "tokens")); + try std.testing.expect(!@hasField(Model.Input, "token_count")); + try std.testing.expect(!@hasField(@FieldType(Model.Input, "weights"), "scale")); + + const Baked = Model.Runner(.{ .weights, .bias }).BakedInput; + try std.testing.expect(@hasField(Baked, "weights")); + try std.testing.expect(@hasField(Baked, "bias")); + try std.testing.expect(!@hasField(Baked, "tokens")); + + const Runtime = Model.Runner(.{ .weights, .bias }).NonBakedInput; + try std.testing.expect(!@hasField(Runtime, "weights")); + try std.testing.expect(!@hasField(Runtime, "bias")); + try std.testing.expectEqual(Buffer, @FieldType(Runtime, "tokens")); + + try std.testing.expectEqual(*Buffer, @FieldType(Model.Output, "hidden")); + try std.testing.expectEqual( + *mem.Bufferized(@FieldType(Outputs, "cache")), + @FieldType(Model.Output, "cache"), + ); + try std.testing.expect(!@hasField(Model.Output, "metadata")); +} + +test "Exe raw calls leave donated input handle cleanup to the caller" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + return value.add(weight).reuseBuffer(value); + } + }; + const value: Tensor = .init(.{2}, .i32); + const weight: Tensor = .init(.{2}, .i32); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ value, weight }, .{}); + defer exe.deinit(); + try std.testing.expectEqualSlices(u32, &.{0}, exe.donated_input_indices); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + args.set(.{ value_buffer, weight_buffer }); + exe.call(io, args, &results, .{ .wait = wait }); + var output = results.get(Buffer); + defer output.deinit(); + + // The donated handle is still valid for cleanup even though its contents are deleted. + for (value_buffer._shards.constSlice()) |shard| { + try std.testing.expect(shard.isDeleted(platform.pjrt_api)); + } + value_buffer.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); + } +} + +test "FnExe runner replaces donated buffers with and without waiting" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { weight: Tensor, value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.add(inputs.weight).reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const weight: Tensor = .init(.{2}, .i32); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); + defer exe.deinit(); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); + defer runner.deinit(allocator); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{}); + try std.testing.expectEqual([2]i32{ 11, 22 }, try value_buffer.getValue([2]i32, io)); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = true }); + try std.testing.expectEqual([2]i32{ 21, 42 }, try value_buffer.getValue([2]i32, io)); +} + +test "FnExe runner replaces unchanged donated buffers" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .value = value }}); + defer exe.deinit(); + try std.testing.expectEqualSlices(u32, &.{0}, exe.raw.donated_input_indices); + + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = wait }); + try std.testing.expectEqual([2]i32{ 1, 2 }, try value_buffer.getValue([2]i32, io)); + } +} diff --git a/zml/floats.zig b/zml/floats.zig index 538722cd..689c22a8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -8,8 +8,8 @@ fn FloatConversionHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -20,7 +20,7 @@ fn FloatConversionHelpers(Float: type) type { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); + const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast @@ -29,16 +29,15 @@ fn FloatConversionHelpers(Float: type) type { const vf32: Float32 = @bitCast(f); const exponent: i16 = @as(i16, vf32.exponent) - exp_off; - const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); + const overflow = std.math.isInf(f) or exponent > std.math.maxInt(@FieldType(Float, "exponent")); if (overflow) { @branchHint(.unlikely); return if (@hasDecl(Float, "inf")) if (vf32.sign == 0) Float.inf else Float.minus_inf else if (@hasDecl(Float, "nan")) Float.nan - else b: { - break :b if (vf32.sign == 0) Float.max else Float.min; - }; + else + std.debug.panic("{} doesn't have infinite representations", .{Float}); } return if (exponent <= 0) @@ -107,8 +106,8 @@ fn FloatHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { - if (!std.mem.eql(u8, field.name, expected_name)) + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) @compileError(err_msg); } } @@ -271,10 +270,8 @@ test "Float8E4" { }) |Float8T| { try testCustomFloat(Float8T, test_case_e4); try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); - if (@hasDecl(Float8T, "inf")) { - try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); - try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); - } + + if (@hasDecl(Float8T, "inf")) @compileError("Float8E4 don't have ±∞ representations"); } } @@ -505,6 +502,25 @@ pub const Float4E2M1 = packed struct(u4) { } try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); } + + pub const Packed = packed struct(@Int(.unsigned, @bitSizeOf(Float4E2M1) * 2)) { + x: Float4E2M1, + y: Float4E2M1, + + pub fn fromF32(x: f32, y: f32) Packed { + return .{ .x = .fromF32(x), .y = .fromF32(y) }; + } + + pub fn toF32(xy: Packed) [2]f32 { + return .{ xy.x.toF32(), xy.y.toF32() }; + } + + pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try xy.x.formatNumber(w, n); + try w.writeByte(','); + try xy.y.formatNumber(w, n); + } + }; }; pub fn floatCast(T: type, x: anytype) T { @@ -528,7 +544,7 @@ pub fn isInf(x: anytype) bool { } if (!@hasDecl(Float, "inf")) return false; - const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); + const FBits = @Int(.unsigned, @bitSizeOf(Float)); const remove_sign = ~@as(FBits, 0) >> 1; return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } diff --git a/zml/io.zig b/zml/io.zig index 9538987b..ab8d8e6e 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -2,48 +2,64 @@ const std = @import("std"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); -pub const VFS = @import("io").VFS; +pub const VFS = @import("vfs"); const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; +const CreateOptions = @import("platform.zig").CreateOptions; +const Exe = @import("exe.zig").Exe; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Placement = Sharding.Placement; +const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { - registry: *safetensors.TensorRegistry, - id_map: std.AutoHashMapUnmanaged(usize, *safetensors.Tensor), - allocator: std.mem.Allocator, + pub const Binding = struct { + tensors: []*safetensors.Tensor, + transformed: bool, + memory: Memory.Kind = .default, + }; - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { + registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, + id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); return .{ .registry = registry, - .id_map = .empty, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), + .id_to_sources = .empty, .allocator = allocator, + .arena = arena, }; } pub fn deinit(self: *TensorStore) void { - self.id_map.deinit(self.allocator); + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); } - fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { - const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; - - const gop = try self.id_map.getOrPut(self.allocator, id); + fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { + const gop = try self.id_to_sources.getOrPut(self.allocator, id); if (gop.found_existing) { - stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); + stdx.debug.panic("Id {} already has associated sources", .{id}); } - errdefer self.id_map.removeByPtr(gop.key_ptr); + errdefer self.id_to_sources.removeByPtr(gop.key_ptr); - gop.value_ptr.* = tensor_desc_ptr; + gop.value_ptr.* = sources; } fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { @@ -51,9 +67,19 @@ pub const TensorStore = struct { return tensor_desc_ptr; } + fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { + const entry = self.getPtrFromKey(key) orelse return null; + + const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); + copy.* = entry.*; + + return copy; + } + fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { - const tensor_desc_ptr = self.id_map.get(id) orelse return null; - return tensor_desc_ptr; + const sources = self.id_to_sources.get(id) orelse return null; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + return sources.tensors[0]; } pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { @@ -61,15 +87,39 @@ pub const TensorStore = struct { } pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const tensor_desc = self.id_map.get(id) orelse return error.NotFound; + const sources = self.id_to_sources.get(id) orelse return error.NotFound; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return safetensors.TensorReader.init(io, tensor_desc.*, buffer, .{}); + return sources.tensors[0].reader(io, buffer, .{}); + } + + pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); + } + + pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { + const entry_ptr = self.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; } pub fn view(self: *TensorStore) View { return .{ .store = self }; } + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + pub const View = struct { store: *TensorStore, @@ -96,7 +146,7 @@ pub const TensorStore = struct { pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); return .{ .store = self.store, @@ -107,7 +157,7 @@ pub const TensorStore = struct { pub fn withLayer(self: *const View, index: usize) View { var buffer: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }) catch unreachable; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); return .{ .store = self.store, @@ -116,56 +166,115 @@ pub const TensorStore = struct { }; } - fn prefix(self: *const View) ?[]const u8 { + pub fn prefix(self: *const View) ?[]const u8 { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } pub fn hasKey(self: *const View, subkey: []const u8) bool { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); return for (self.store.registry.tensors.keys()) |k| { if (std.mem.startsWith(u8, k, key)) break true; } else false; } - pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); + } + + const resolved_mesh = self.store.mesh(mesh_); + if (@TypeOf(partitioning) == Sharding.Partitioning) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); + } + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| + .parse(p_tags, resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); + } + + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const source = self.store.dupeSource(key) orelse return null; - const ptr = self.store.getPtrFromKey(key) orelse return null; + const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); + errdefer self.store.arena.allocator().free(sources); + sources[0] = source; - const tensor: Tensor = .fromShape(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; + var shape = source.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } - pub fn createTensor(self: View, subkey: []const u8) Tensor { - return self.maybeCreateTensor(subkey).?; + /// Accepts "null" as tags + fn parseTags(tags: anytype) ?[]const Shape.Tag { + return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); } - pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { + const arena = self.store.arena.allocator(); + + var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); + defer tensor_list.deinit(arena); + var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + for (sources) |subkey| { + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const tensor = self.store.dupeSource(key) orelse return null; + tensor_list.appendAssumeCapacity(tensor); + } - const ptr = self.store.getPtrFromKey(key) orelse return null; - ptr.shape = ptr.shape.withTags(tagz); + const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; + errdefer arena.free(tensors); - const tensor: Tensor = .fromShape(ptr.shape); - self.store.bindIdToKey(key, tensor.id) catch unreachable; + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); return tensor; } - pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { - return self.maybeCreateTensorWithTags(subkey, tagz).?; - } - pub fn getShape(self: View, subkey: []const u8) ?Shape { var buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getShape(key); } pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { @@ -173,15 +282,14 @@ pub const TensorStore = struct { const key = if (opts.no_prefix) subkey else b: { - break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); }; - const entry_ptr = self.store.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; + return self.store.getShape(key); } pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { var key_buffer: [256]u8 = undefined; - const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); return self.store.getReader(key, io, buffer); } @@ -197,9 +305,300 @@ pub const TensorStore = struct { } return count_; } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } }; }; +pub const Loader = struct { + allocator: std.mem.Allocator, + platform: *const Platform, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + pinned_buffer_pools: []mem.DynamicBufferPool, + group: stdx.Io.LimitedGroup, + bytes_loaded: std.atomic.Value(usize) = .init(0), + delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, + + pub const Opts = struct { + pub const default: Opts = .{ + .parallelism = 1, + .dma_chunks = 2, + .dma_chunk_size = 4096, + }; + parallelism: usize, + dma_chunks: usize, + dma_chunk_size: usize, + }; + + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { + const pool_count = platform.devices.len; + const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); + errdefer allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(allocator, device); + } + + const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); + errdefer allocator.free(buffer_pools); + for (buffer_pools) |*pool_| { + pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); + } + errdefer for (buffer_pools, 0..) |*pool_, i| { + pool_.deinit(dma_allocators[i].allocator()); + }; + + return .{ + .allocator = allocator, + .platform = platform, + .dma_allocators = dma_allocators, + .dma_chunk_size = opts.dma_chunk_size, + .pinned_buffer_pools = buffer_pools, + .group = .init(opts.parallelism), + }; + } + + pub fn deinit(self: *Loader) void { + self.delivered.deinit(self.allocator); + for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); + self.allocator.free(self.pinned_buffer_pools); + self.allocator.free(self.dma_allocators); + } + + pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { + return self.group.await(io); + } + + pub const LoadOpts = struct { + progress: ?*std.Progress.Node = null, + }; + + pub const LoadError = error{TransformedTensorNotDelivered}; + + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { + const tensor_count = meta.count(Tensor, model); + + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(buffers, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { + flattened_buffers_[i] = buffer; + } + }.call, .{flattened_buffers}); + + const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { + flattened_tensors_[i] = tensor; + } + }.call, .{flattened_tensors}); + + // Refuse before scheduling anything, so a failure leaves no task in flight. + var missing: usize = 0; + for (flattened_tensors) |tensor| { + const sources = store.getSourcesById(tensor.id) orelse continue; + if (!sources.transformed or self.delivered.contains(tensor.id)) { + continue; + } + + missing += 1; + logNotDelivered(arena.allocator(), tensor, sources.tensors); + } + + if (missing != 0) { + return error.TransformedTensorNotDelivered; + } + + for (flattened_tensors, flattened_buffers) |tensor, buffer| { + if (store.getSourcesById(tensor.id)) |sources| { + if (sources.transformed) continue; + } + + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); + } + } + + fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { + const max_names = 8; + var names: std.Io.Writer.Allocating = .init(arena); + for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { + names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; + } + + if (sources.len > max_names) { + names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; + } + + log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); + } + + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); + } + + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { + const sources = store.getSourcesById(tensor.id) orelse { + std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); + return; + }; + stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); + + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { + log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); + unreachable; + }; + } + + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { + log.err("Failed to load tensor {s}: {}", .{ source.name, e }); + loaded.* = false; + return; + }; + loaded.* = true; + } + + fn loadSingleInner( + self: *Loader, + io: std.Io, + source: *safetensors.Tensor, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + opts: LoadOpts, + ) !void { + var reader = try source.reader(io, &.{}, .{}); + defer reader.deinit(); + + var writer = try MemoryWriter.init( + self.allocator, + io, + self.platform, + self.pinned_buffer_pools, + self.dma_allocators, + self.dma_chunk_size, + shape, + buffer, + memory, + ); + defer writer.deinit(self.allocator); + + const scale = 1024; + + if (opts.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + writer.setProgress(&node); + defer writer.setProgress(null); + var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); + const total = try reader.interface.streamRemaining(&progress_writer.interface); + try progress_writer.interface.flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } else { + const total = try reader.interface.streamRemaining(writer.interface()); + try writer.interface().flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } + } + + pub fn loadExecute( + self: *Loader, + arena: std.mem.Allocator, + io: std.Io, + tensor: Tensor, + buffer: *Buffer, + store: *const TensorStore, + exe: *const Exe, + opts: LoadOpts, + ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { + const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; + const buffers = try arena.alloc(Buffer, sources.len); + const loaded = try arena.alloc(bool, sources.len); + @memset(loaded, false); + defer for (buffers, loaded) |*b, l| if (l) b.deinit(); + + var node = if (opts.progress) |progress| p: { + var writer = std.Io.Writer.Allocating.init(arena); + print: { + writer.writer.writeAll("Running executable on ") catch break :print; + for (sources, 0..) |source, i| { + writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; + } + } + break :p progress.start(writer.written(), 1); + } else null; + defer if (node) |*n| n.end(); + + for (sources, 0..) |source, i| { + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); + } + self.group.await(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + }; + + if (std.mem.findScalar(bool, loaded, false)) |_| { + return error.LoadFailed; + } + + var args = try exe.args(arena); + var results = try exe.results(arena); + + args.set(.{buffers}); + exe.call(io, args, &results, .{ .wait = true }); + + buffer.* = results.get(Buffer); + try self.delivered.put(self.allocator, tensor.id, {}); + } +}; + +test "Loader.loadExecute cleans up donated and non-donated inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = @import("testing.zig").env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); + defer loader.deinit(); + + inline for (.{ false, true }) |donate| { + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + const sum = value.add(weight); + return if (donate) sum.reuseBuffer(value) else sum; + } + }; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); + defer exe.deinit(); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); + + var output: Buffer = undefined; + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expect(loader.delivered.contains(tensor.id)); + } +} + pub const ProgressWriter = struct { inner: *std.Io.Writer, progress: *std.Progress.Node, @@ -238,7 +637,8 @@ pub const ProgressWriter = struct { pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { self.interface.buffer = self.inner.buffer; self.interface.end = self.inner.end; - self.total += (len_pre - self.interface.end) + total; + const drained_pre = len_pre -| self.interface.end; + self.total += drained_pre + total; self.progress.setCompletedItems(self.total / self.scale); } @@ -279,10 +679,25 @@ pub const MemoryWriter = union(enum) { direct: DirectMemoryWriter, buffered: BufferedMemoryWriter, - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { - return switch (memory.platform.target) { - .cuda => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, - .rocm, .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !MemoryWriter { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .furiosa => .{ + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), + }, + .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) + else + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, }; } @@ -299,20 +714,27 @@ pub const MemoryWriter = union(enum) { .buffered => self.buffered.deinit(allocator), } } + + pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { + switch (self.*) { + .direct => self.direct.setProgress(progress), + .buffered => {}, + } + } }; pub const BufferedMemoryWriter = struct { io: std.Io, - memory: *const Memory, + platform: *const Platform, shape: Shape, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, + .platform = platform, .shape = shape, - .memory = memory, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -331,36 +753,22 @@ pub const BufferedMemoryWriter = struct { } } - fn flush_(w: *std.Io.Writer) !void { - const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - const pjrt_api = self.memory.platform.pjrt_api; - - const args: pjrt.Client.BufferFromHostBufferArgs = .{ - .data = @ptrCast(self.interface.buffer), - .buffer_type = pjrtx.bufferTypeFromDtype(self.shape.dtype()), - .dims = self.shape.dims(), - .byte_strides = self.shape.computeByteStrides().constSlice(), - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - .dst = .{ .memory = self.memory.pjrt_memory }, - }; - const pjrt_buffer, const event = try self.memory.platform.pjrt_client.bufferFromHostBuffer(pjrt_api, args); - self.buffer.* = Buffer.fromPjrtBuffers(self.memory.platform, self.shape, &.{pjrt_buffer}); - if (event) |ev| { - defer ev.deinit(pjrt_api); - try ev.await(pjrt_api, self.io); - } else { - try self.buffer.await(self.io); - } - } - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - flush_(w) catch return std.Io.Writer.Error.WriteFailed; + const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + self.buffer.* = Buffer.from( + self.io, + self.platform, + self.shape, + @ptrCast(self.interface.buffer), + .{ .wait = true }, + ) catch return std.Io.Writer.Error.WriteFailed; } }; -pub const DirectMemoryWriter = struct { +const DirectShardWriter = struct { const EventContext = struct { - self: *DirectMemoryWriter, + self: *DirectShardWriter, err: ?*pjrt.Error = null, pjrt_event: *pjrt.Event, event: std.Io.Event = .unset, @@ -372,7 +780,7 @@ pub const DirectMemoryWriter = struct { memory: *const Memory, pool: *mem.DynamicBufferPool, total: usize, - buffer: *Buffer, + pjrt_buffer: *pjrt.Buffer, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, offset: usize = 0, @@ -381,7 +789,7 @@ pub const DirectMemoryWriter = struct { flip_flop: u1 = 0, events_contexts: [2]?EventContext = @splat(null), - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !DirectMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( memory.platform.pjrt_api, @@ -390,9 +798,12 @@ pub const DirectMemoryWriter = struct { .memory = memory.pjrt_memory, }, ); + errdefer transfer_manager.deinit(memory.platform.pjrt_api); - const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; - buffer.* = Buffer.fromPjrtBuffers(memory.platform, shape, &.{pjrt_buffer}); + const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); + errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); + + const buf = try pool.get(allocator, io); return .{ .allocator = allocator, @@ -400,54 +811,82 @@ pub const DirectMemoryWriter = struct { .memory = memory, .pool = pool, .total = shape.byteSize(), - .buffer = buffer, + .pjrt_buffer = pjrt_buffer, .transfer_manager = transfer_manager, .interface = .{ - .buffer = try pool.get(allocator, io), - .end = 0, + .buffer = buf[0..@min(buf.len, shape.byteSize())], .vtable = &.{ .drain = drain, + .flush = flush, + .rebase = rebase, }, }, }; } - pub fn deinit(self: *DirectMemoryWriter) void { + pub fn deinit(self: *DirectShardWriter) void { self.transfer_manager.deinit(self.memory.platform.pjrt_api); } - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - _ = data; // autofix - _ = splat; // autofix - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.offset >= self.total) { - return std.Io.Writer.Error.WriteFailed; + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + + const chunk = data[0]; + if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; + + const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; + if (needs_fresh_buffer) { + try self.interface.flush(); } + @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); + self.interface.end += chunk.len; + + const buffer_full = self.interface.end == self.interface.buffer.len; + if (buffer_full) { + try self.interface.flush(); + } + + return chunk.len; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + try self.flushBuffered(); + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + try self.interface.flush(); + } + + fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { + if (self.offset >= self.total) return; + const pjrt_api = self.memory.platform.pjrt_api; const current_buffer = self.interface.buffer; const buffered = self.interface.buffered(); - const sliced_buffer = self.interface.buffered()[0..@min(buffered.len, self.total - self.offset)]; - const written = sliced_buffer.len; - const is_last = (self.offset + sliced_buffer.len) >= self.total; - const transfer_event = self.transfer_manager.transferData( - pjrt_api, - 0, - sliced_buffer, - @intCast(self.offset), - is_last, - ) catch |err| { + const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; + if (slice.len == 0) return; + const is_last = (self.offset + slice.len) >= self.total; + + const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { log.err("error when transferring data to device: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; + const ctx = &self.events_contexts[@intCast(self.flip_flop)]; ctx.* = .{ .self = self, .buffer = current_buffer, .pjrt_event = transfer_event, }; + transfer_event.onReady(pjrt_api, EventContext, struct { fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); @@ -489,113 +928,816 @@ pub const DirectMemoryWriter = struct { } } else { self.interface.end = 0; - self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { + const buf = self.pool.get(self.allocator, self.io) catch |err| { log.err("unable to get a new buffer from the pool: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; + self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; } self.flip_flop ^= 1; - self.offset += written; - - return 0; + self.offset += slice.len; } }; -pub const LoadOpts = struct { - parallelism: usize, - store: *const TensorStore, - progress: ?*std.Progress.Node = null, - dma_chunks: usize, - dma_chunk_size: usize, - total_bytes: ?*usize = null, +const ShardProgress = struct { + const scale: usize = 1024; + + node: std.Progress.Node, + label: [32]u8 = undefined, + completed: usize = 0, + + fn set(self: *ShardProgress, completed: usize) void { + self.completed = completed; + self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); + } }; -pub fn load( - comptime ModelType: type, - model: *const ModelType, +// Dispatch planning bridges two different orders: +// +// 1. Placement traversal emits byte ranges per shard writer, in device order. +// That order is convenient for asking "what bytes belong to this shard?", +// but it does not match how a tensor file is read. +// +// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter +// must therefore consume dispatch spans in increasing global byte offset. +// +// We first collect placement spans, sort them by global byte range, then fold +// identical ranges into one primary dispatch span plus mirror writers. Identical +// ranges represent replicated shard data: the reader provides those bytes once, +// the primary writer receives them zero-copy, and mirrors receive a copy. +const DispatchSpans = struct { + const DispatchSpan = struct { + start: usize, + end: usize, + primary_writer: usize, + mirror_writer_start: usize, + mirror_writer_len: usize, + }; + + const PlacementSpan = struct { + writer_index: usize, + start: usize, + len: usize, + order: usize, + }; + + spans: []DispatchSpan, + mirror_writers: []usize, + + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(&shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); + + var placement_span_count: usize = 0; + for (0..ordered_devices.len) |device_index| { + placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); + } + + var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); + defer placement_spans.deinit(allocator); + + const byte_strides = shape.computeByteStrides(); + + for (0..ordered_devices.len) |writer_index| { + appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); + } + + std.debug.assert(placement_spans.items.len == placement_span_count); + + var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); + errdefer spans.deinit(allocator); + + var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); + errdefer mirror_writers.deinit(allocator); + + try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); + + const spans_ = try spans.toOwnedSlice(allocator); + errdefer allocator.free(spans_); + + const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); + errdefer allocator.free(mirror_writers_); + + return .{ + .spans = spans_, + .mirror_writers = mirror_writers_, + }; + } + + fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { + allocator.free(self.spans); + allocator.free(self.mirror_writers); + } + + fn deduplicateByRange( + allocator: std.mem.Allocator, + placement_spans: []PlacementSpan, + total_bytes: usize, + spans: *std.ArrayList(DispatchSpan), + mirror_writers: *std.ArrayList(usize), + ) !void { + const SortContext = struct { + fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { + if (lhs.start != rhs.start) return lhs.start < rhs.start; + if (lhs.len != rhs.len) return lhs.len < rhs.len; + return lhs.order < rhs.order; + } + }; + + std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); + + var i: usize = 0; + var cursor: usize = 0; + while (i < placement_spans.len) { + const span = placement_spans[i]; + if (span.start != cursor) return error.NonContiguousShardPlacement; + + const mirror_writer_start = mirror_writers.items.len; + var j = i + 1; + while (j < placement_spans.len) : (j += 1) { + const mirror = placement_spans[j]; + if (mirror.start != span.start or mirror.len != span.len) break; + try mirror_writers.append(allocator, mirror.writer_index); + } + + try spans.append(allocator, .{ + .start = span.start, + .end = span.start + span.len, + .primary_writer = span.writer_index, + .mirror_writer_start = mirror_writer_start, + .mirror_writer_len = j - i - 1, + }); + cursor += span.len; + i = j; + } + + if (cursor != total_bytes) return error.NonContiguousShardPlacement; + } + + fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { + placement_spans.appendAssumeCapacity(.{ + .writer_index = writer_index, + .start = start, + .len = len, + .order = placement_spans.items.len, + }); + } + + fn appendShardPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + shape: Shape, + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + ) void { + if (shape.rank() == 0) { + appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); + return; + } + + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); + } + + fn appendShardAxisPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + axis: usize, + contiguous_axis: usize, + base_start: i64, + ) void { + const slice = slices[axis]; + if (slice.size == 0) return; + + if (axis == contiguous_axis) { + const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); + const span_len: usize = @intCast(slice.size * byte_strides[axis]); + appendPlacementSpan(placement_spans, writer_index, span_start, span_len); + return; + } + + var i: i64 = 0; + while (i < slice.size) : (i += 1) { + const child_start = base_start + (slice.start + i) * byte_strides[axis]; + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); + } + } + + fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { + if (shape.rank() == 0) return 1; + + const contiguous_axis = contiguousSliceAxis(shape, slices); + var count: usize = 1; + for (slices[0..contiguous_axis]) |slice| { + count *= @intCast(slice.size); + } + return count; + } + + fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { + var axis = shape.rank() - 1; + while (axis > 0) { + const slice = slices[axis]; + if (slice.start != 0 or slice.size != shape.dim(axis)) break; + axis -= 1; + } + return axis; + } +}; + +// Direct load writer state machine. +// +// std.Io.Reader +// | +// v +// DirectMemoryWriter.interface.buffer +// | aliases active primary DirectShardWriter DMA buffer +// v +// commitWindow() +// |-- primary: advance DirectShardWriter.interface.end +// |-- mirrors: copy committed bytes with writeAll() +// |-- boundaries: flush full shard buffers / chunk fence +// v +// DirectShardWriter.flushBuffered() +// | +// v +// PJRT AsyncHostToDeviceTransferManager +// +// The public writer never owns a host staging buffer: it exposes a visible +// prefix of a real shard DMA buffer. The visible prefix may extend beyond the +// current dispatch span to coalesce reader requests, but `commitWindow` must +// scatter those bytes before submitting or rotating the active buffer. The +// first active-primary segment is zero-copy; crossed spans and mirrors treat +// the active buffer as scratch and copy into their own shard writers. +// +// `byte_cursor` is the global tensor position. Shard writer `interface.end` is +// local to the current shard DMA buffer. +pub const DirectMemoryWriter = struct { + allocator: std.mem.Allocator, + shard_writers: []DirectShardWriter, + // Global stream-order spans produced from placement; this is the routing table for committed bytes. + dispatch_spans: DispatchSpans, + // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. + span_index: usize = 0, + // Global tensor byte offset already scattered into shard writers. + byte_cursor: usize = 0, + // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. + active_writer_index: usize, + // Start offset of new reader bytes inside the active shard writer buffer. + window_start: usize, + // Logical maximum public alias window before forcing a cross-shard flush fence. + dma_chunk_size: usize, + shard_progress: ?[]ShardProgress = null, + interface: std.Io.Writer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !DirectMemoryWriter { + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(&placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); + var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); + errdefer allocator.free(shard_writers); + + var initialized: usize = 0; + errdefer for (shard_writers[0..initialized]) |*writer| { + writer.deinit(); + }; + + var pjrt_buffers: Buffer.Shards = .empty; + for (ordered_devices, 0..) |device, i| { + const pool = &pools[device.id]; + const shard_dma_allocator = dma_allocators[device.id].allocator(); + const pjrt_mem = platform.devices[device.id].memory(memory).?; + + shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); + initialized += 1; + + pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); + } + + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); + + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); + errdefer dispatch_spans.deinit(allocator); + + const first_span = dispatch_spans.spans[0]; + const first_writer = &shard_writers[first_span.primary_writer]; + const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); + + return .{ + .allocator = allocator, + .shard_writers = shard_writers, + .dispatch_spans = dispatch_spans, + .active_writer_index = first_span.primary_writer, + .window_start = first_writer.interface.end, + .dma_chunk_size = dma_chunk_size, + .interface = .{ + .buffer = first_writer.interface.buffer[0..first_window], + .end = 0, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectMemoryWriter) void { + self.setProgress(null); + for (self.shard_writers) |*writer| { + writer.deinit(); + } + self.allocator.free(self.shard_writers); + self.dispatch_spans.deinit(self.allocator); + } + + pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { + const parent = progress orelse { + const states = self.shard_progress orelse return; + for (states) |*s| { + s.node.end(); + } + self.allocator.free(states); + self.shard_progress = null; + return; + }; + + std.debug.assert(self.shard_progress == null); + const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; + for (states, self.shard_writers, 0..) |*state, writer, i| { + state.completed = 0; + const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; + const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; + state.node = parent.start(label, total_items); + state.node.setCompletedItems(0); + } + self.shard_progress = states; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + try self.commitWindow(); + if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + + const consumed = consumed: { + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + const copied = self.writeWindow(bytes); + n += copied; + + if (copied < bytes.len) break :consumed n; + } + + const pattern = data[data.len - 1]; + switch (pattern.len) { + 0 => {}, + 1 => { + const dest = self.interface.buffer[self.interface.end..]; + const count = @min(dest.len, splat); + @memset(dest[0..count], pattern[0]); + + self.interface.end += count; + n += count; + }, + else => for (0..splat) |_| { + const copied = self.writeWindow(pattern); + n += copied; + + if (copied < pattern.len) break; + }, + } + + break :consumed n; + }; + + try self.commitWindow(); + return consumed; + } + + fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { + const dest = self.interface.buffer[self.interface.end..]; + const n = @min(dest.len, bytes.len); + @memcpy(dest[0..n], bytes[0..n]); + self.interface.end += n; + return n; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + // Commit the active public window if one remains + if (self.span_index < self.dispatch_spans.spans.len) { + try self.commitWindow(); + } + + for (self.shard_writers, 0..) |*shard_writer, i| { + try shard_writer.interface.flush(); + if (self.shard_progress) |states| { + states[i].set(shard_writer.total); + } + } + + self.interface = .failing; + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + + try self.commitWindow(); + if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; + } + + // `interface.buffer` is a clipped alias of the active primary shard DMA + // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; + // this function scatters that visible range through dispatch spans. + // + // The leading segment for `active_writer_index` is already in the right + // DMA buffer and is committed by advancing that shard writer. Later spans + // use the same bytes as scratch and are copied before any full buffers are + // submitted or the public alias is rotated. + fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + // All dispatch spans are consumed; no shard DMA buffer remains valid to + // expose as the public alias until flush marks the writer as failing. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Only bytes after `window_start` are new reader writes. Bytes before + // it may already be pending in the active shard writer. + const source = self.interface.buffer[self.window_start..self.interface.end]; + if (source.len == 0) return; + + // The visible alias window may cross many dispatch spans. Walk the + // global cursor and route each subrange to its primary and mirrors. + var consumed: usize = 0; + while (consumed < source.len) { + if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; + + const span = self.dispatch_spans.spans[self.span_index]; + if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; + + const n = @min(source.len - consumed, span.end - self.byte_cursor); + const chunk = source[consumed..][0..n]; + + // The first span in the active alias is already in the primary DMA + // buffer. Later ranges use the active buffer as scratch and must be + // copied into their real destination writer. + if (span.primary_writer == self.active_writer_index and consumed == 0) { + self.shard_writers[span.primary_writer].interface.end += n; + } else { + const primary_writer = &self.shard_writers[span.primary_writer]; + if (span.primary_writer == self.active_writer_index) { + @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); + primary_writer.interface.end += n; + } else { + try primary_writer.interface.writeAll(chunk); + } + } + if (self.shard_progress) |states| { + states[span.primary_writer].set(states[span.primary_writer].completed + n); + } + + const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; + for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { + const mirror_writer = &self.shard_writers[mirror_writer_index]; + if (mirror_writer_index == self.active_writer_index) { + @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); + mirror_writer.interface.end += n; + } else { + try mirror_writer.interface.writeAll(chunk); + } + if (self.shard_progress) |states| { + states[mirror_writer_index].set(states[mirror_writer_index].completed + n); + } + } + + self.byte_cursor += n; + consumed += n; + if (self.byte_cursor == span.end) { + self.span_index += 1; + } + } + + // All scratch copies are complete now, so full DMA buffers may be + // submitted without invalidating bytes still needed for scatter. + for (self.shard_writers) |*writer| { + if (writer.interface.end == writer.interface.buffer.len) { + try writer.interface.flush(); + } + } + + if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { + for (self.shard_writers) |*writer| { + try writer.interface.flush(); + } + } + + // The last committed window may finish the tensor exactly. Leave an + // empty public window so any further writes fail through std.Io. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Rotate the public alias to the primary writer for the next stream + // span. `window_start` preserves that writer's already-buffered prefix + // so only newly advanced bytes are scattered on the next commit. + const next_span = self.dispatch_spans.spans[self.span_index]; + const next_writer = &self.shard_writers[next_span.primary_writer]; + self.active_writer_index = next_span.primary_writer; + self.window_start = next_writer.interface.end; + + // Publish the next public alias. Its backing memory is the next primary + // shard buffer, but the visible length is allowed to coalesce across + // upcoming spans until a buffer or tensor boundary. + const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; + const buffer_remaining = next_writer.interface.buffer.len - self.window_start; + const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); + const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; + const tensor_remaining = total - self.byte_cursor; + const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); + if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; + + self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; + self.interface.end = self.window_start; + } +}; + +const DirectMemoryWriterDeviceTest = struct { + const WriteMode = enum { + stream_remaining, + writable_slice_greedy, + }; + + pub const Scenario = struct { + name: []const u8, + create_options: CreateOptions, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + write_mode: WriteMode = .stream_remaining, + writable_slice_min_len: usize = 128, + pool_chunks: usize = 4, + pool_chunk_size: usize = 1 << 20, + memory: Memory.Kind = .default, + }; + allocator: std.mem.Allocator, io: std.Io, - platform: *const Platform, - opts: LoadOpts, -) !Bufferized(ModelType) { - var bufferized = try mem.bufferize(allocator, ModelType, model); - const first_device = platform.devices[0]; // Temporary until sharding is re-exposed - const dma_alloc: mem.DmaAllocator = .init(allocator, &first_device); - var buffer_pool: mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); - defer buffer_pool.deinit(dma_alloc.allocator()); + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { + var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; + defer platform.deinit(self.allocator, self.io); - const Ctx = struct { - allocator: std.mem.Allocator, - dma_allocator: std.mem.Allocator, - pinned_buffer_pool: *mem.DynamicBufferPool, - io: std.Io, - buffers: []*Buffer, - store: *const TensorStore, - memory: *const Memory, - group: stdx.Io.LimitedGroup, - total: std.atomic.Value(usize) = .init(0), - progress: ?*std.Progress.Node, - }; - var walk_ctx: Ctx = .{ - .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), - .store = opts.store, - .allocator = allocator, - .dma_allocator = dma_alloc.allocator(), - .pinned_buffer_pool = &buffer_pool, - .io = io, - .memory = first_device.memory(.default), - .progress = opts.progress, - .group = .init(opts.parallelism), - }; - defer allocator.free(walk_ctx.buffers); + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + std.log.warn("mesh: {f}", .{mesh}); + try self.runDirectMemoryWriter( + platform, + scenario.shape.withPartitioning(&mesh, partitioning), + scenario.write_mode, + scenario.writable_slice_min_len, + scenario.pool_chunks, + scenario.pool_chunk_size, + scenario.memory, + ); + } - defer if (opts.total_bytes) |total_bytes_ptr| { - total_bytes_ptr.* = walk_ctx.total.load(.monotonic); - }; + fn runDirectMemoryWriter( + self: DirectMemoryWriterDeviceTest, + platform: *const Platform, + shape: Shape, + write_mode: WriteMode, + writable_slice_min_len: usize, + pool_chunks: usize, + pool_chunk_size: usize, + memory: Memory.Kind, + ) !void { + const slice = try Slice.alloc(self.allocator, shape); + defer slice.free(self.allocator); - meta.forEachVisit(&bufferized, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, ctx: *Ctx) void { - ctx.buffers[i] = buffer; + for (slice.items(f32), 0..) |*e, i| { + e.* = @as(f32, @floatFromInt(i)); } - }.call, .{&walk_ctx}); - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, ctx: *Ctx) void { - ctx.group.concurrent(ctx.io, struct { - fn call(i_: usize, tensor_: *const Tensor, ctx_: *Ctx) !void { - var reader = ctx_.store.getReaderById(tensor_.id, ctx_.io, &.{}) catch unreachable; - defer reader.deinit(); + const pool_count = platform.devices.len; + const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); + defer self.allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(self.allocator, device); + } - var memory_writer = MemoryWriter.init( - ctx_.dma_allocator, - ctx_.io, - ctx_.memory, - ctx_.pinned_buffer_pool, - reader.tensor.shape, - ctx_.buffers[i_], - ) catch unreachable; - defer memory_writer.deinit(ctx_.dma_allocator); + const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); + defer self.allocator.free(pools); + for (pools) |*pool| { + pool.* = .init(pool_chunks, pool_chunk_size); + } + defer for (pools, 0..) |*pool, i| { + pool.deinit(dma_allocators[i].allocator()); + }; - const scale = 1024; + var written_buffer: Buffer = undefined; + var writer: DirectMemoryWriter = try .init( + self.allocator, + self.io, + platform, + pools, + dma_allocators, + pool_chunk_size, + shape, + &written_buffer, + memory, + ); + defer writer.deinit(); + defer written_buffer.deinit(); - if (ctx_.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - var progress_writer: ProgressWriter = .init(memory_writer.interface(), &node, .{ .scale = scale }); - const total = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; - progress_writer.interface.flush() catch unreachable; - _ = ctx_.total.fetchAdd(total, .monotonic); - } else { - const total = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; - memory_writer.interface().flush() catch unreachable; - _ = ctx_.total.fetchAdd(total, .monotonic); - } + switch (write_mode) { + .stream_remaining => { + var reader: std.Io.Reader = .fixed(slice.constData()); + const streamed = try reader.streamRemaining(&writer.interface); + try std.testing.expectEqual(slice.constData().len, streamed); + }, + .writable_slice_greedy => { + var offset: usize = 0; + while (offset < slice.constData().len) { + const min_len = @max(@as(usize, 1), writable_slice_min_len); + const dest = try writer.interface.writableSliceGreedy(min_len); + const to_write = @min(dest.len, slice.constData().len - offset); + if (to_write == 0) return std.Io.Writer.Error.WriteFailed; + @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); + writer.interface.advance(to_write); + offset += to_write; } - }.call, .{ i, tensor, ctx }) catch unreachable; + }, } - }.call, .{&walk_ctx}); - walk_ctx.group.await(io) catch unreachable; - return bufferized; + try writer.interface.flush(); + try written_buffer.await(self.io); + + var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); + defer written_slice.free(self.allocator); + try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); + } +}; + +test "DirectMemoryWriter: replicated tensor" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "replicated_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "batch_model_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + }, .{ .batch = .batch, .model = .model }); +} + +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_folded_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + }, .{ .model = .model }); +} + +test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto_writable_slice", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .write_mode = .writable_slice_greedy, + .writable_slice_min_len = 64, + .pool_chunk_size = 1024, + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 3D topology folded model + replicated batch" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "topology_3d_folded_model", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .cpu = .{ .device_count = 8 }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + }, .{ .batch = .replicated, .model = .model }); +} + +test "MemoryWriter can produce a host pinned buffer" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "host_pinned", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .memory = .host_pinned, + }, .{ .batch = .replicated, .model = .model }); } diff --git a/zml/io/io.zig b/zml/io/io.zig deleted file mode 100644 index 78c59e21..00000000 --- a/zml/io/io.zig +++ /dev/null @@ -1 +0,0 @@ -pub const VFS = @import("vfs.zig").VFS; diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig deleted file mode 100644 index a282371c..00000000 --- a/zml/io/vfs.zig +++ /dev/null @@ -1,398 +0,0 @@ -const std = @import("std"); - -const vfs = @import("vfs"); - -const log = std.log.scoped(.@"zml/io/vfs"); - -pub const VFS = struct { - pub const File = vfs.File; - pub const HTTP = vfs.HTTP; - pub const HF = vfs.HF; - pub const S3 = vfs.S3; - const VFSBase = vfs.VFSBase; - - const Handles = std.ArrayList(Handle); - const ClosedHandles = std.ArrayList(u32); - - const CWD_HANDLE: u32 = 0; - - const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - - backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, - handles: Handles = .{}, - closed_handles: ClosedHandles = .{}, - - base: VFSBase, - - pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { - const base = VFSBase.init(base_io); - - var handles: Handles = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); - - return .{ - .allocator = allocator, - .handles = handles, - .base = base, - }; - } - - pub fn deinit(self: *VFS) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - self.backends.deinit(self.allocator); - } - - pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.backends.put(self.allocator, scheme, backend); - } - - pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.backends.remove(scheme); - } - - pub fn io(self: *VFS) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *VFS) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, &self.handles.items[idx] }; - } - return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *VFS, idx: u32) !void { - if (idx == CWD_HANDLE) return; - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { - self.mutex.lockUncancelable(self.base.inner); - const handle = &self.handles.items[@intCast(file.handle)]; - self.mutex.unlock(self.base.inner); - - return .{ handle, self.getBackend(handle.backend_idx) }; - } - - fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (std.meta.eql(dir, std.Io.Dir.cwd())) return &self.handles.items[CWD_HANDLE]; - return &self.handles.items[@intCast(dir.handle)]; - } - - fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { - if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; - } - - fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; - } - - fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } - } else { - const handle = self.getDirHandle(dir); - if (handle.backend_idx) |backend_idx| { - return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; - } else { - return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; - } - } - } - - fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle, const backend = self.getFileHandle(o.file); - return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, - .data = o.data, - } }); - }, - .device_io_control, .file_write_streaming => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.OpenError.Unexpected; - }; - const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(fs_dir.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx) }; - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir stat : {any}", .{err}); - return std.Io.Dir.StatError.Unexpected; - }; - return backend.vtable.dirStat(backend.userdata, dir_); - } - - pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.StatFileError.Unexpected; - }; - return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.AccessError.Unexpected; - }; - return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.AccessError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx), .flags = handle.flags }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (dirs) |dir| { - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for closing dir : {any}", .{err}); - continue; - }; - backend.vtable.dirClose(backend.userdata, &.{dir_}); - self.closeHandle(@intCast(dir.handle)) catch unreachable; - } - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.RealPathError.Unexpected; - }; - - const original_dir = reader.dir; - reader.dir = dir_; - defer reader.dir = original_dir; - - return backend.vtable.dirRead(backend.userdata, reader, entries); - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.RealPathError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); - } - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { - log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); - return std.Io.Dir.RealPathFileError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); - } - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle, const backend = self.getFileHandle(file); - backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); - - const original_src_handle = reader.file; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; - - return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, _ = self.getFileHandle(reader.file); - - const original_src_handle = reader.file; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer reader.file = original_src_handle; - - return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - - if (self.getScheme(handle.backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); - } - } -}; diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig deleted file mode 100644 index 605fc8c6..00000000 --- a/zml/io/vfs/index.zig +++ /dev/null @@ -1,9 +0,0 @@ -const log = std.log.scoped(.@"zml/io/vfs/vfs"); - -const std = @import("std"); - -pub const File = @import("file.zig").File; -pub const HF = @import("hf.zig").HF; -pub const HTTP = @import("http.zig").HTTP; -pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md new file mode 100644 index 00000000..81a35cf4 --- /dev/null +++ b/zml/kernels/flash_attention/README.md @@ -0,0 +1,287 @@ +# Flash attention semantics + +This directory exposes causal scaled dot-product attention through +[`zml.flash_attention.attention`](flash_attention.zig). This document describes +the attention operation, the FlashAttention algorithm, and ZML's current semantic +contract. Shapes below are logical tensor shapes. Sharding describes ownership +of those tensors across devices; platform-specific execution layouts are outside +the scope of this document. + +## The attention operation + +For one sequence and one head, let `Q` contain `Lq` query vectors, and let `K` and +`V` contain `Lk` key/value vectors. Each query is compared with each permitted key; +the resulting probabilities weight the corresponding values: + +```text +S = scale * Q @ transpose(K) + M +P = softmax(S, axis=keys) +O = P @ V +``` + +The usual scale is `1 / sqrt(D)`, where `D` is the query/key head dimension. +An additive mask `M` is zero for allowed pairs and negative infinity for forbidden +pairs. Softmax normalizes each query row independently over keys, so a masked key +has zero probability. Heads and batch elements are independent. ZML's +[`sdpa`](../../nn.zig) implements this reference computation. + +Attention in general permits a value dimension `Dv` different from the query/key +dimension `D`; the output then has dimension `Dv`. This module uses the same +`.hd = D` for queries, keys, values, and output. + +FlashAttention can evaluate either unrestricted attention or masked attention. +Causality is a choice of mask, rather than an inherent property of the algorithm. +Other attention APIs may offer local windows, additive score biases, a custom +scale, and dropout on the probabilities during training. These options change +the operation and must be matched when comparing results. The +[upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#how-to-use-flashattention) +documents such options; ZML's interface here exposes a narrower operation. + +## What makes attention "flash" + +FlashAttention computes the same mathematical attention result while avoiding +storage of the complete `Lq × Lk` score and probability matrices. It processes +query/key/value tiles and combines partial results using an online softmax. +This reduces intermediate memory and memory traffic. Dense attention still +requires `O(Lq * Lk * D)` arithmetic per head; for self-attention with `Lq = Lk`, +that remains quadratic in sequence length. "Exact" means the attention formula +is preserved, with ordinary floating-point rounding differences. See the +[FlashAttention paper](https://arxiv.org/abs/2205.14135). + +For each query row, online softmax maintains a maximum `m`, an exponential sum +`l`, and an unnormalized weighted-value vector `u`. Given scores `s_j` for a new +tile of permitted keys, update: + +```text +m_new = max(m, max_j(s_j)) +a = exp(m - m_new) +p_j = exp(s_j - m_new) +l_new = a * l + sum_j(p_j) +u_new = a * u + sum_j(p_j * V_j) +O = u / l # after all tiles +``` + +Initialize with `m = -infinity`, `l = 0`, and `u = 0`; skip tiles with no permitted +keys. Rescaling the previous accumulators preserves a single normalization across +all tiles. Independently normalizing each tile and adding its output would give +a different result. The [FlashAttention-2 paper, sections 2–3](https://arxiv.org/html/2307.08691v1) +describes this recurrence and how training can recompute probability tiles during +the backward pass instead of retaining the full matrix. This module exposes only +the attention output, with no backward entry point or returned probabilities. + +## Heads and head dimensions + +`D` counts features in **one head**. It is distinct from the number of heads and +the sequence length. In a conventional multi-head projection, +`D = hidden_size / Hq`; models with an explicit head dimension need not follow +that relation. The query projection has width `Hq * D`, while each key/value +projection has width `Hkv * D`. + +Typical dimensions include: + +| Head dimension | Context | +| --- | --- | +| `64` | A common transformer head size; also the size used by this module's shared tests. | +| `128` | Another common size; Qwen2.5-7B uses `3584 / 28 = 128`. | +| `256` | Used by models such as Gemma 2 2B and 9B. | +| `512` | Used by Gemma 4 global attention layers, for example in Gemma 4 31B. | + +The [FlashAttention-2 paper](https://arxiv.org/html/2307.08691v1) describes typical +head sizes around 64–128. The model examples follow the +[Qwen2.5-7B configuration](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct/raw/main/config.json) +and [Gemma configurations](https://github.com/google/gemma_pytorch/blob/main/gemma/config.py). +Gemma can also use different head dimensions within one model: the +[Gemma 4 31B configuration](https://huggingface.co/google/gemma-4-31B/blob/main/config.json) +specifies `head_dim = 256` for local attention and `global_head_dim = 512` for +global attention. Use the dimension configured for the specific layer. +These are model dimensions, not a list of supported kernel sizes or a mathematical +requirement that `D` be a power of two. + +The number of query heads `Hq` may differ from the number of key/value heads +`Hkv`. ZML uses contiguous groups of query heads: + +```text +G = Hq / Hkv +kv_head(h) = floor(h / G) +``` + +`Hq` must be divisible by `Hkv`, and keys and values must have the same head count. +This covers multi-head attention (MHA: `Hq = Hkv`), grouped-query attention +(GQA: `1 < Hkv < Hq`), and multi-query attention (MQA: `Hkv = 1`). For example, +with `Hq = 64` and `Hkv = 8`, query heads 0–7 use KV head 0, heads 8–15 use KV +head 1, and so on. Sharing keys and values does not merge query heads: each query +head still has its own scores, normalization, and output. + +## ZML inputs and output + +The entry point is: + +```zig +attention(q, k, v, token_index, metadata, parameters) +``` + +The reference contract uses these tagged shapes, with an optional `.b` batch axis: + +| Argument | Unbatched shape | Batched shape | Meaning | +| --- | --- | --- | --- | +| `q` | `{ .q = Lq, .h = Hq, .hd = D }` | `{ .b = B, .q = Lq, .h = Hq, .hd = D }` | Queries to evaluate. | +| `k`, `v` | `{ .k = Lk, .h = Hkv, .hd = D }` | `{ .b = B, .k = Lk, .h = Hkv, .hd = D }` | Keys and values in sequence order. | +| `token_index` | Scalar | `{ .b = B }` or scalar | Zero-based position of the first query in the key sequence; a scalar applies to the whole batch. | +| Result | Same as `q` | Same as `q` | Weighted values for every query head. | + +Use matching floating-point dtypes for `q`, `k`, and `v`; the output retains the +query dtype. The shared tests use `bf16` tensors and `u32` positions. `metadata` +and `parameters` provide execution configuration rather than additional masks, +score biases, or model projections. + +### Causality and query positions + +Let `t = token_index` for a sequence. Query row `i` represents position `t + i`, +and key row `j` represents position `j`. The inclusive causal rule is: + +```text +allowed(i, j) = (j <= t + i) +M[i, j] = 0 if allowed(i, j), otherwise -infinity +``` + +The reference path constructs a square causal mask over `Lk` positions, gathers +`Lq` consecutive rows starting at `t`, and passes that mask to `sdpa`. Valid inputs +require `Lq > 0`, `t >= 0`, and `t + Lq <= Lk`. The test helper checks the upper +bound; callers must satisfy it rather than rely on out-of-range gather behavior. +Each valid query can attend to at least its own key, so fully masked rows are +outside this contract. + +| Use case | Query positions | Keys visible to query row `i` | +| --- | --- | --- | +| Initial prefill | `t = 0`, usually `Lq = Lk` | `0` through `i`. | +| Chunked prefill | `Lq > 1`, starting at `t` | `0` through `t + i`. | +| Single-token decode | `Lq = 1`, current token at `t` | `0` through `t`, including the current token. | + +For `Lq = 2`, `Lk = 5`, and `t = 2`, the mask is: + +```text + key position + 0 1 2 3 4 +query at 2 1 1 1 0 0 +query at 3 1 1 1 1 0 +``` + +Here `1` means permitted. `Lk` may be the capacity of a preallocated KV buffer; +it does not determine the current query position. All permitted KV entries must +already be populated. Positions after the last query are masked, and the call +does not append to or update a KV cache. + +For unequal query and key lengths, causal alignment must be specified explicitly. +The [upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#21-change-behavior-of-causal-flag) +uses bottom-right alignment: `j <= i + Lk - Lq`. ZML's reference rule agrees with +that convention when `t = Lk - Lq`. An explicit `token_index` also describes +queries earlier in a larger buffer, as in the example above. + +### Sharding and local attention + +Sharding changes where attention is evaluated, not its mathematical result. +The global output still has the query's tagged shape and dtype. Each local call +must receive the matching queries, keys, values, and positions for the batch +elements and heads it owns. + +| Axis or input | Sharding semantics | +| --- | --- | +| Batch `.b` | Sequences are independent. Q, K, V, and the output must assign the same batch elements to each shard. | +| Heads `.h` | Each shard must have the KV heads needed by its query heads, preserving `kv_head(h) = floor(h / G)`. K and V must have matching head ownership. | +| Query sequence `.q` | Query rows are independent once the required KV entries are available. Their causal positions remain positions in the original sequence. | +| Key sequence `.k` | Softmax normalizes over all permitted keys for a query, including keys on other shards if the sequence is distributed. Separate local softmax outputs cannot simply be added or averaged. | +| Features `.hd` | Every score uses the complete head dimension and the global scale `1 / sqrt(D)`. Splitting features requires combining partial dot products before softmax. | +| `token_index` | A scalar applies to every shard. Per-sequence positions follow the batch partition and are replicated across head shards of the same sequence. Head or batch sharding does not change their values. | + +For head sharding without KV replication, a partition count `P` must divide +`Hkv`, and therefore also `Hq`. Each shard owns complete, contiguous GQA groups: +`Hq_local = Hq / P`, `Hkv_local = Hkv / P`, and the local ratio remains `G`. +For example, with `Hq = 32`, `Hkv = 8`, and two shards, the first shard owns query +heads 0–15 and KV heads 0–3; the second owns query heads 16–31 and KV heads 4–7. +Each query still uses exactly the same KV head as in the unsharded computation. +MQA has only one KV head, so distributing its query heads over multiple devices +requires replicating that KV head rather than dividing it into independent heads. + +The local-kernel execution contract keeps `.q`, `.k`, and `.hd` complete on each +shard, with parallelism across batches and/or complete head groups. Thus each +local call has the full sequence coordinates and feature vectors needed by the +causal mask, dot products, and normalization. The mathematical possibilities for +sequence and feature sharding above do not imply that these local kernels +implement distributed sequence or feature reductions. + +An **open** partition means placement is unresolved; it does not mean the tensor +must be replicated. Operations such as cache slicing and RoPE can reopen +partition annotations even when their inputs were sharded. Before entering a +local kernel, resolve compatible partitions on a common mesh and align Q, K, V, +and positions. The KV layout anchors this alignment: an explicit KV head +partition is retained, while an open head partition can use a compatible mesh +axis whose partition count divides `Hkv`. Preserve explicit batch ownership and +do not assign the same device partition independently to both batch and heads. +An explicit replicated KV head axis remains replicated; open heads can also +remain replicated when no compatible head partition is available. Automatic +partition selection is an execution choice, not an additional attention argument. + +The output follows the **resolved** query partition. Its partition annotations +need not match the incoming query's unresolved annotations, and aligning operands +may require communication. In particular, unnecessarily resolving open KV heads +to replicated can gather the full cache on every attention call even though the +head groups could be evaluated independently. This changes cost, not attention +semantics, and does not modify the caller's KV cache. + +Splitting KV work into internal tiles or partitions is distinct from sharding the +input tensors across devices. Either way, combining partial attention states +must preserve the single softmax normalization. For disjoint key subsets with +states `(m_r, l_r, u_r)` as defined above: + +```text +m = max_r(m_r) +l = sum_r(exp(m_r - m) * l_r) +u = sum_r(exp(m_r - m) * u_r) +O = u / l +``` + +An empty subset contributes zero (`m_r = -infinity`, `l_r = 0`, `u_r = 0`). +Every valid query has at least one permitted key overall, so its merged +normalization is nonempty. + +### Scope and current conformance + +The public call always requests causal attention with scale `1 / sqrt(D)` and no +dropout. It has no arguments for an arbitrary mask, padding lengths, packed +sequence boundaries, a local window, ALiBi, score soft-capping, a softmax sink, or +a custom scale. The lower-level `sdpa` and `causalAttnMask` helpers expose some +additional options, but this wrapper does not pass them through. Separate batch +elements can have different `token_index` values; concatenating independent +sequences along `.k` does not isolate them from each other. + +Q/K/V projections, positional transformations such as RoPE, and any Q/K +normalization must happen before this call. Head concatenation and the output +projection happen afterward. A model's use of a familiar head dimension alone +does not establish compatibility with this operation. + +The reference contract above is broader than current kernel coverage. In +particular, nonzero-offset multi-query prefill and the optional batch dimension +are not uniformly supported by the current kernels; an existing prefill path +requires equal query/key lengths and assumes positions begin at zero. Treat +these as current conformance gaps, not alternative definitions of causality. + +The tests in [tests.zig](tests.zig) compare results +against `sdpa` with the gathered causal mask. The original shared cases cover +`D = 64`, MHA and GQA, +single-token decode at full and partial context lengths, batched decode with +different positions, and an eight-query chunk beginning at position 56 in a +64-position KV buffer. Additional conformance cases exercise head dimensions +through `512`, MQA, `bf16`/`f16`/`f32`, initial and offset prefill, partial tiles, +transposed tagged layouts, scalar batch positions, head/batch sharding, open +query/KV partitions, and split decode with empty partitions and cache tails. These +cases are enabled for kernels that support them; their presence does not imply +uniform coverage across every kernel. FP32 results are also checked against an +independent host calculation using FP64 arithmetic. + +Numerical agreement is tolerance-based. The reference evaluates softmax in +`f32`, then converts probabilities to the query dtype before multiplying by +values. Different accumulation orders and rounding points can produce small +differences even when the mathematical operation is identical. The shared tests +use absolute tolerance `5e-3`, relative tolerance `1e-2`, relative epsilon `1e-3`, +and a minimum close fraction of `0.99`; these are test criteria rather than a +universal error bound. diff --git a/zml/kernels/flash_attention/cute/BUILD.bazel b/zml/kernels/flash_attention/cute/BUILD.bazel new file mode 100644 index 00000000..b83a642e --- /dev/null +++ b/zml/kernels/flash_attention/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + ], +) diff --git a/zml/kernels/flash_attention/cute/README.md b/zml/kernels/flash_attention/cute/README.md new file mode 100644 index 00000000..8c55d2c8 --- /dev/null +++ b/zml/kernels/flash_attention/cute/README.md @@ -0,0 +1,145 @@ +# CuTe flash attention scheduling + +The [parent README](../README.md) defines attention and sharding semantics. +This document describes decode scheduling in this implementation and the +FlashInfer heuristics that inform it. Scheduling changes parallelism and +intermediate storage, not the attention formula or public arguments. + +## What FlashInfer does + +These references are pinned to FlashInfer commit +`2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a`; FlashInfer has multiple execution paths, +so these rules describe the cited CUDA decode paths rather than every backend. + +[Single-sequence decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/decode.cuh#L705-L744) +uses one attention launch when the live sequence has at most 256 tokens or no +scratch buffer is supplied. Otherwise it queries the compiled kernel's resident +blocks per SM and the device's SM count, then computes: + +```text +grid_budget = resident_blocks_per_SM * SM_count +split_budget = floor(grid_budget / KV_heads) +chunk_tokens = max(ceil(live_tokens / split_budget), 256) +splits = ceil(live_tokens / chunk_tokens) +``` + +It launches partial attention over these chunks followed by `MergeStates`. +The merge preserves a single softmax normalization across the chunks. + +[Batched paged decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L75-L98) +uses the per-request lengths to search for a chunk size whose total block count +fits the occupancy budget. Its +[work estimator](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L174-L205) +disables splitting if `batch_size * KV_heads` already fills that budget. The +minimum chunk is `max(floor(128 / page_size), 1)` pages. Without CUDA graphs it +also avoids splitting when partitioning adds no requests. Graph execution can +retain a padded split schedule to keep launch and allocation sizes stable. + +## Adaptation to this kernel + +[`decodeSplits`](attention.zig) runs after sharding has established each device's +local dimensions. A block owns at most four query heads belonging to one KV +head, so our unsplit decode block count is: + +```text +G = local_query_heads / local_KV_heads +base_blocks = local_batch * local_KV_heads * ceil(G / 4) +``` + +Counting query-head groups matters: a large GQA group already launches multiple +blocks per KV head in this kernel. Using global batch or head counts would +underestimate the need for splitting after partitioning across devices. + +XLA compiles and owns the CUDA function after this planner runs. We cannot call +CUDA's occupancy calculator on that function here. Instead, the planner uses +PJRT's `core_count` and `shared_memory_per_block_optin` attributes and a +static limit of twelve 128-thread blocks per SM: + +```text +padded_D = next_power_of_two(max(D, 32)) +shared_bytes_per_block = 2 * 16 * padded_D * sizeof(dtype) # K and V tiles +blocks_per_SM = max(1, min(12, floor(shared_budget / shared_bytes_per_block))) +grid_budget = 4 * SM_count * blocks_per_SM +``` + +The per-block shared-memory limit serves as a conservative shared-memory budget; +it is not a measurement of SM capacity or compiled-kernel occupancy. The +twelve-block limit is a scheduling estimate, not a measured occupancy or a +hardware constant. It provides enough resident work to hide latency in the +128-thread kernel; register usage and other hardware limits can reduce actual +occupancy. Four scheduling waves provide additional work for the serial +KV-tile loop; this is a measured adaptation, whereas the cited FlashInfer CUDA +planner targets one resident grid. Larger heads and FP32 reduce the budget through their shared-memory requirements. For +multiple visible devices, the smallest reported SM count and shared-memory +budget are used. Missing or invalid attributes disable splitting. + +The selection rules are: + +1. Prefill (`Lq > 1`) remains unsplit. +2. A cache capacity of at most 64 tokens remains unsplit. +3. If `base_blocks >= grid_budget`, keep one partition per block's head group. +4. Otherwise set `split_budget = ceil(grid_budget / base_blocks)`, + `minimum_chunk = 16` for capacities through 256, otherwise `32`, + `chunk = max(minimum_chunk, ceil(cache_capacity / split_budget))`, and + `splits = ceil(cache_capacity / chunk)`. +5. If the result is one split, launch attention directly into the output. Other + cases allocate FP32 partial states and launch attention followed by a merge. + +The chunk floors are one or two of this kernel's 16-key tiles. They adapt +FlashInfer's minimum-work-per-partition heuristic to our serial per-tile score +loop. Applying its literal 128/256-token floors and a smaller block budget +regressed several measured cases. The workload-based budget and split/merge +structure follow FlashInfer, but these constants are specific to this kernel. +The split budget rounds up a partial wave: rounding 32 partitions down to 31, +for example, can add a whole tile to every block. The grid budget is therefore +a target, and the launched grid may exceed it by less than one split's worth +of `base_blocks`. + +## Capacity and live lengths + +Our graph receives `token_index` on device, potentially with a different value +for each batch element. Planning uses static cache capacity to bound the grid +and scratch allocation without reading positions back to the host. At runtime, +each sequence distributes its **live** prefix of `token_index + 1` keys evenly +across the planned splits. Therefore the actual chunk can be smaller than the +planning floor. Empty partitions contribute zero to the merge. + +This is not FlashInfer's host-side ragged-length search. A short live sequence +in a large cache can still use the split path, and the merge launch remains in +the graph even when only one partition has keys. The schedule does not change +from token to token, so it can be reused during generation and graph capture. + +Partial states contain the weighted-value accumulator, maximum, and exponential +sum in FP32. Their storage is +`4 * local_batch * splits * local_query_heads * (D + 2)` bytes. Reducing splits +as the batch grows reduces both scratch traffic and merge work. Scores are +scaled by `log2(e) / sqrt(D)` so attention and merging use base-2 exponentials. + +## Validation and timing + +The existing attention harness covers GQA/MQA, different positions within a +batch, partial tiles, empty partitions, sharding, three dtypes, and head +dimensions through 512. Planner tests cover the short-cache boundary, decreasing +split counts as batch size grows, resource limits, and extra query-head groups. +Runtime checks include batches through 2048 at the 64/65-token scheduling +boundary; they establish correctness, not throughput at that batch size. + +An opt-in benchmark in [cute.zig](cute.zig) compares the previous 32-split policy +with the planner at batches 1, 4, 16, and 64, and full contexts of 128, 2048, and +8192 tokens. It uses BF16, 16 query heads, four KV heads, and `D = 128` per device. +Sixteen dependent attention calls amortize host overhead; reported microseconds +per attention call are the median of 16 samples after three warmups. This is a +repeated-cache kernel benchmark, not end-to-end model throughput. + +```sh +bazel test //zml:test --//platforms:cuda=true --//platforms:cpu=false \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + --@rules_zig//zig/settings:zigopt=zml.test.zml \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + '--@rules_zig//zig/settings:zigopt=CuTe decode' \ + --test_env=CUDA_VISIBLE_DEVICES=0 \ + --test_env=ZML_CUTE_ATTENTION_BENCHMARK=1 --test_output=all +``` + +`CUTE_DECODE_BENCH` rows contain batch, context length, policy, split count, and +microseconds per call. Timing is not a correctness assertion. diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig new file mode 100644 index 00000000..ec0fce16 --- /dev/null +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -0,0 +1,818 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const Value = cute.Value; + +pub const Config = struct { + batch: i64, + query_length: i64, + key_length: i64, + query_heads: i64, + kv_heads: i64, + head_dim: i64, + dtype: cute.DType, + splits: i64 = 1, + /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. + causal: bool = true, +}; + +/// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA +/// function, so this is an estimated resource budget, not an occupancy query. +pub const DecodeResources = struct { + multiprocessors: i64, + shared_bytes: i64, +}; + +pub fn decodeSplits(q: zml.Shape, k: zml.Shape, resources: DecodeResources) i64 { + const query_length = q.dim(.q); + const key_length = k.dim(.k); + const num_heads = q.dim(.h); + const num_kv_heads = k.dim(.h); + const head_dim = q.dim(.hd); + const batch = if (q.hasTag(.b) != null) q.dim(.b) else 1; + const dtype = q.dtype(); + + if (query_length != 1 or key_length <= 64) return 1; + if (resources.multiprocessors <= 0 or resources.shared_bytes <= 0) return 1; + const group = @divExact(num_heads, num_kv_heads); + const blocks_per_sequence = num_kv_heads * @divTrunc(group + 3, 4); + const base_blocks = batch * blocks_per_sequence; + var padded_dim: i64 = 32; + while (padded_dim < head_dim) padded_dim *= 2; + const bytes_per_element: i64 = if (dtype == .f32) 4 else 2; + const shared_per_block = 2 * tile_keys * padded_dim * bytes_per_element; + const blocks_per_sm = @max(1, @min(12, @divTrunc(resources.shared_bytes, shared_per_block))); + // Multiple scheduling waves keep the serial tile loop supplied with work. + // The measured tradeoff and distinction from CUDA occupancy are in README. + const grid_budget = 4 * resources.multiprocessors * blocks_per_sm; + if (base_blocks >= grid_budget) return 1; + // Round up a partial wave: flooring can leave SMs idle and add a whole + // extra KV tile to every block (e.g. 31 partitions versus 32). + const split_budget = @divTrunc(grid_budget + base_blocks - 1, base_blocks); + // One tile for short caches, two otherwise. FlashInfer's larger chunk + // floor is tuned for a different per-block KV processing rate. + // With static dense shapes, capacity bounds the scratch space and grid; + // each invocation balances its live prefix over these partitions on device. + const min_chunk: i64 = if (key_length <= 256) tile_keys else 2 * tile_keys; + const chunk = @max(min_chunk, @divTrunc(key_length + split_budget - 1, split_budget)); + return @divTrunc(key_length + chunk - 1, chunk); +} + +test "CuTe decode planner respects workload and resource limits" { + const resources: DecodeResources = .{ .multiprocessors = 170, .shared_bytes = 101376 }; + var q = zml.Shape.init(.{ .b = 1, .q = 1, .h = 16, .hd = 128 }, .bf16); + var k = zml.Shape.init(.{ .b = 1, .k = 2048, .h = 4, .hd = 128 }, .bf16); + try std.testing.expectEqual(64, decodeSplits(q, k, resources)); + k = k.setDim(.k, 64); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + k = k.setDim(.k, 65); + try std.testing.expectEqual(5, decodeSplits(q, k, resources)); + k = k.setDim(.k, 2048); + var previous: i64 = 64; + for ([_]i64{ 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 }) |batch| { + q = q.setDim(.b, batch); + k = k.setDim(.b, batch); + const splits = decodeSplits(q, k, resources); + try std.testing.expect(splits <= previous); + try std.testing.expect(splits >= 1); + try std.testing.expect(splits == 1 or (splits - 1) * batch * 4 < 8160); + previous = splits; + } + try std.testing.expectEqual(1, previous); + q = q.setDim(.b, 16); + k = k.setDim(.b, 16); + q = q.setDim(.hd, 512); + k = k.setDim(.hd, 512); + q = q.withDtype(.f32); + k = k.withDtype(.f32); + try std.testing.expectEqual(11, decodeSplits(q, k, resources)); + q = q.setDim(.q, 7); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + q = q.setDim(.q, 1); + try std.testing.expectEqual(1, decodeSplits(q, k, .{ .multiprocessors = 0, .shared_bytes = 0 })); + // More query-head groups already create more independent CTAs. + q = q.setDim(.b, 1); + k = k.setDim(.b, 1); + q = q.setDim(.hd, 128); + k = k.setDim(.hd, 128); + q = q.withDtype(.bf16); + k = k.withDtype(.bf16); + q = q.setDim(.h, 1024); + try std.testing.expectEqual(32, decodeSplits(q, k, resources)); +} + +pub const block: [3]i32 = .{ 128, 1, 1 }; +const tile_keys = 16; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "flash_attention_cute", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(cfg.head_dim > 0 and cfg.head_dim <= 512); + inline for (.{ 1, 2, 4, 8, 16 }) |fragments| { + if (cfg.head_dim <= fragments * 32) return runSized(b, cfg, fragments); + } + unreachable; +} + +/// Four warps own four (query position, query head) rows sharing one KV head. +/// Only a KV tile lives in shared memory; maxima, sums, and outputs stay in +/// registers. All lanes participate in shuffles, including padded rows/features. +fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void { + const padded_dim = fragments * 32; + const split = cfg.splits > 1; + const out_shape = if (split) &[_]i64{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 } else &[_]i64{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim }; + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim } } }, + .k = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .v = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .positions = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.batch} } }, + .out = .{ .tensor = .{ .dtype = if (split) .f32 else cfg.dtype, .shape = out_shape } }, + }); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const group = @divExact(cfg.query_heads, cfg.kv_heads); + const batch = b.blockIdx().y.div(cfg.kv_heads); + const kv_head = b.blockIdx().y.rem(cfg.kv_heads); + const first_row = b.blockIdx().x.mul(4); + const row = first_row.add(tid.div(32)); + // Padded warps compute a valid dummy row and suppress their final stores. + const query = row.div(group).minimum(cfg.query_length - 1); + const head = kv_head.mul(group).add(row.rem(group)); + const start = a.positions.get(.{batch}); + const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); + const live_end = if (cfg.causal) + start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) + else + b.constant(.i32, cfg.key_length); + // Partition the live cache, rather than its capacity, so decode uses all + // blocks even when only a prefix of the cache is populated. + const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); + const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); + const end = if (split) begin.add(chunk).minimum(live_end) else live_end; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + + var queries: [fragments]Value = undefined; + inline for (0..fragments) |f| { + const d = lane.add(f * 32); + var valid = b.openIfElse(d.lt(cfg.head_dim), .{.f32}); + { + valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); + } + { + valid.yieldElse(.{zero}); + } + queries[f] = valid.results[0]; + } + + const layout = b.makeLayout(&.{ tile_keys, padded_dim }, .{ .stride = &.{ padded_dim, 1 } }); + const keys = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + const values = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + var initial: [fragments + 2]Value = @splat(zero); + initial[0] = neg_inf; + var tiles = b.openFor(begin, end, tile_keys, initial); + { + + // Zero-fill partial tiles and feature padding before the CTA-wide barrier. + var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); + { + const tile_key = copy.iv.div(padded_dim); + const d = copy.iv.rem(padded_dim); + const key = tiles.iv.add(tile_key); + var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype, cfg.dtype }); + { + valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); + } + { + valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); + } + keys.set(.{ tile_key, d }, valid_load.results[0]); + values.set(.{ tile_key, d }, valid_load.results[1]); + copy.close(); + } + b.syncThreads(); + + // Each lane keeps one score; dot products reduce across the feature lanes. + var scores = b.openFor(0, tile_keys, 1, .{neg_inf}); + { + var dot = zero; + inline for (0..fragments) |f| { + dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); + } + inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); + const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); + const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); + scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); + } + var maximum = scores.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + maximum = maximum.maximum(tiles.carried[0]); + const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); + const probability = b.exp2Approx(scores.results[0].sub(maximum)); + var sum = probability; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); + sum = sum.add(tiles.carried[1].mul(rescale)); + + var accumulators: [fragments]Value = undefined; + inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); + var weighted = b.openFor(0, tile_keys, 1, accumulators); + { + const weight = b.shuffleIdx(probability, weighted.iv); + inline for (0..fragments) |f| { + accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); + } + weighted.yield(accumulators); + } + // No warp may overwrite the tile before every warp has consumed it. + b.syncThreads(); + var next: [fragments + 2]Value = undefined; + next[0] = maximum; + next[1] = sum; + inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; + tiles.yield(next); + } + + inline for (0..fragments) |f| { + const col = lane.add(f * 32); + var valid_store = b.openIf(row.lt(cfg.query_length * group).bitAnd(col.lt(cfg.head_dim))); + { + if (split) { + a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); + } else { + a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); + } + valid_store.close(); + } + } + if (split) { + var stats = b.openIf(row.lt(cfg.query_length * group).bitAnd(lane.eq(0))); + { + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); + stats.close(); + } + } +} + +/// Merge unnormalized partial attention states in FP32. Empty partitions have +/// m=-inf, l=0, u=0 and contribute zero, including at the first decode token. +pub const Merge = cute.Kernel(Config, .{ + .name = "flash_attention_cute_merge", + .inputs = &.{"partials"}, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, cfg: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.batch, cfg.splits, cfg.query_heads, cfg.head_dim + 2 } } }, + .out = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_heads, cfg.head_dim } } }, + }); + const batch = b.blockIdx().y; + const head = b.blockIdx().x; + const lane = b.threadIdx().x.rem(32); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + // Every warp reduces the partition statistics, then independently handles + // a contiguous range of output features. + var maxima = b.openFor(lane, cfg.splits, 32, .{neg_inf}); + { + maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); + } + var maximum = maxima.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + const weights = b.makeTensor(b.allocSmem(.f32, cfg.splits, 16), b.makeLayout(&.{cfg.splits}, .{})); + var stats = b.openFor(b.threadIdx().x, cfg.splits, 128, .{}); + { + weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); + stats.close(); + } + b.syncThreads(); + var features = b.openFor(b.threadIdx().x, cfg.head_dim, 128, .{}); + { + var sum = b.openFor(0, cfg.splits, 1, .{ zero, zero }); + { + const weight = weights.get(.{sum.iv}); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), + }); + } + a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); + features.close(); + } +} + +// ==================== Prefill ==================== +// +// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 +// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one +// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and +// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, +// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied +// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. +// +// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared +// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas +// caps at 48 KiB before sm_120. + +pub const Prefill = cute.Program(Config, .{ + .name = "flash_attention_cute_prefill", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = prefillProgram, +}); + +const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; + +/// Keys per tile (FA2's kBlockN). +const prefill_keys = 64; + +/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. +/// Same as FA2 we only support 16-bit inputs +pub fn prefillSupported(cfg: Config) bool { + const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; + const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; + return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; +} + +/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. +fn prefillRowTiles(head_dim: i64) i64 { + return if (head_dim <= 128) 2 else 1; +} + +/// Queries per block (FA2's kBlockM). +fn prefillRows(cfg: Config) i64 { + return 4 * 16 * prefillRowTiles(cfg.head_dim); +} + +/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. +fn prefillGrid(cfg: Config) [3]i32 { + return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; +} + +fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(prefillSupported(cfg)); + switch (cfg.head_dim) { + inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { + inline .bf16, .f16 => |dt| { + b.beginFunction(prefill_kernel_name, .cuda_kernel); + try prefillSized(b, cfg, d, dt); + b.endFunction(block); + + b.beginFunction(Prefill.name, .host); + const host = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const grid = prefillGrid(cfg); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, + .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), + .stream = b.cudaStream(), + }); + const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); + }, + else => unreachable, + }, + else => unreachable, + } +} + +/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and +/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. +fn fmax(b: *B, x: Value, y: Value) Value { + return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; +} + +fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { + var result: [n]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + } + return result; +} + +/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. +fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { + const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), + .constraints = "=r,=r,=r,=r,r", + .has_side_effects = true, + }, b.loc())); + return unpackStruct(b, 4, result, .i32); +} + +/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). +fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), + .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop +/// carries at most 64 values). +fn packAcc(b: *B, values: [4]Value) Value { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn unpackAcc(b: *B, value: Value) [4]Value { + var result: [4]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.vectorExtract(value, @intCast(i), .f32); + } + return result; +} + +/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. +/// +/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so +/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses +/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a +/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from +/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant +/// (XORed into the 16-byte vector index). +fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { + return struct { + const row_tiles = prefillRowTiles(d); + const rows = 4 * 16 * row_tiles; + const n = prefill_keys; + const score_tiles = n / 8; + const out_tiles = d / 8; + const row_bytes = d * 2; + /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. + const swizzle_rows = @min(8, d / 8); + const q_bytes = rows * row_bytes; + const kv_bytes = n * row_bytes; + /// Rows covered by one 16-byte cp.async per thread of the block. + const copy_rows = @divExact(128, d / 8); + /// Running max and per-thread partial sum of the two rows of every row tile. + const stats = 2 * row_tiles; + /// Carried by the key loop: maxima, sums, then the packed output accumulators of every + /// (row tile, 8 output features). + const carried = 2 * stats + row_tiles * out_tiles; + const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); + + comptime { + // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. + std.debug.assert(std.math.isPowerOfTwo(copy_rows)); + std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); + } + + b: *B, + cfg: Config, + lane: Value, + warp: Value, + first_query: Value, + start: Value, + q_smem: Value, + k_smem: Value, + v_smem: Value, + k_ptr: Value, + v_ptr: Value, + kv_base: Value, + tiles: Value, + /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. + copy_row: Value, + copy_col: Value, + /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. + copy_row_offset: Value, + copy_vector_offset: Value, + /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. + q_frag_row: Value, + k_frag_row: Value, + v_frag_row: Value, + qv_frag_vector: Value, + k_frag_vector: Value, + + const Self = @This(); + + fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { + const lane = tid.rem(32); + const warp = tid.div(32); + const copy_row = tid.div(d / 8); + const copy_col = tid.rem(d / 8); + const lane8 = lane.rem(8); + // The fragment rows a thread reads are lane % 8 plus multiples of 8. + const lane_xor = lane.rem(swizzle_rows); + return .{ + .b = b, + .cfg = cfg, + .lane = lane, + .warp = warp, + .first_query = first_query, + .start = start, + .q_smem = shared, + .k_smem = shared.add(q_bytes), + .v_smem = shared.add(q_bytes + kv_bytes), + .k_ptr = k_ptr, + .v_ptr = v_ptr, + .kv_base = kv_base, + .tiles = tiles, + .copy_row = copy_row, + .copy_col = copy_col, + .copy_row_offset = copy_row.mul(row_bytes), + .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), + // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). + .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), + // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). + .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), + // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). + .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), + .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), + .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), + }; + } + + /// Destination of the i-th cp.async of this thread in a tile starting at `base`. + fn copyDst(t: Self, base: Value, comptime i: usize) Value { + const xor = ((copy_rows * i) % swizzle_rows) * 16; + const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); + return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); + } + + /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. + fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { + const b = t.b; + const stride = t.cfg.kv_heads * d; + const first_key = key_base.add(t.copy_row); + const safe = src.add(t.kv_base.mul(2)); + const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(n, copy_rows)) |i| { + const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); + const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); + b.cpAsync16(t.copyDst(dst, i), source, valid); + } + } + + fn queryOf(t: Self, mi: usize, h: usize) Value { + return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); + } + + fn accIndex(comptime mi: usize, comptime di: usize) usize { + return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); + } + + /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are + /// fully visible to every query of the block. + fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { + const b = t.b; + const lane = t.lane; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const key_base = tile.mul(n); + b.cpAsyncWaitAll(); + b.syncThreads(); + t.loadTile(t.v_ptr, t.v_smem, key_base); + + // S = Q K^T, k-steps of 16 features. + var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); + inline for (0..d / 16) |ks| { + var qa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); + } + const k_vector = t.k_frag_vector.bitXor(ks * 32); + inline for (0..score_tiles / 2) |np| { + const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); + inline for (0..row_tiles) |mi| { + s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); + s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); + } + } + } + + // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: + // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). + var next: [carried]Value = undefined; + var alpha: [row_tiles][2]Value = undefined; + inline for (0..row_tiles) |mi| { + if (masked) { + inline for (0..score_tiles) |ni| { + inline for (0..4) |j| { + const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); + const in_range = key.lt(t.cfg.key_length); + const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; + s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); + } + } + } + inline for (0..2) |h| { + var row_max = s[mi][0][h * 2]; + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); + } + } + inline for (.{ 2, 1 }) |offset| { + row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); + } + const old_max = carried_in[mi * 2 + h]; + const new_max = fmax(b, old_max, row_max); + // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. + const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); + alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); + next[mi * 2 + h] = new_max; + var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); + sum = sum.add(s[mi][ni][h * 2 + j]); + } + } + next[stats + mi * 2 + h] = sum; + } + } + + // All warps are done with K_j once V_j has landed: start copying K_{j+1}. + b.cpAsyncWaitAll(); + b.syncThreads(); + var prefetch = b.openIf(tile.add(1).lt(t.tiles)); + { + t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); + prefetch.close(); + } + + // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. + var o: [row_tiles][out_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); + inline for (0..4) |j| { + o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); + } + } + } + inline for (0..n / 16) |ks| { + var pa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + pa[mi] = .{ + b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), + b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), + b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), + b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), + }; + } + const v_row = t.v_frag_row.add(ks * 16 * row_bytes); + inline for (0..out_tiles / 2) |dp| { + const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); + inline for (0..row_tiles) |mi| { + o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); + o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); + } + } + } + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + next[accIndex(mi, di)] = packAcc(b, o[mi][di]); + } + } + return next; + } + }; +} + +fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { + const T = PrefillTile(d, dt); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + + const tid = b.threadIdx().x; + const first_query = b.blockIdx().x.mul(T.rows); + const head = b.blockIdx().y; + const batch = b.blockIdx().z; + const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); + // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the + // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and + // `allocSmemView` both become static shared memory instead. + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); + const fields = [_][]const u8{ + std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), + std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), + std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), + }; + const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); + const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); + const q_ptr = b.ptrToInt(a.q, .i64); + const out_ptr = b.ptrToInt(a.out, .i64); + const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; + + // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. + const q_stride = cfg.query_heads * d; + const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); + const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); + + // Causal: the last query of the block sees keys up to start + last_query, and every query of the + // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the + // keys, is masked. + const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); + const tiles = if (cfg.causal) + start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) + else + b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); + const full_tiles = if (cfg.causal) + start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) + else + b.constant(.i32, @divTrunc(cfg.key_length, T.n)); + + const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); + + // Out of range rows are zero-filled and never read from global memory. + { + const first_row = first_query.add(t.copy_row); + const safe = q_ptr.add(q_base.mul(2)); + const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(T.rows, T.copy_rows)) |i| { + const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); + b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); + } + } + t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); + + var initial: [T.carried]Value = undefined; + const zero = b.constant(.f32, 0); + for (initial[0..T.stats]) |*v| { + v.* = b.constant(.f32, -std.math.inf(f32)); + } + for (initial[T.stats .. 2 * T.stats]) |*v| { + v.* = zero; + } + for (initial[2 * T.stats ..]) |*v| { + v.* = packAcc(b, .{ zero, zero, zero, zero }); + } + + var full = b.openFor(0, full_tiles, 1, initial); + { + full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); + } + var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); + { + diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); + } + const results = diagonal.results[0..T.carried].*; + + // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. + inline for (0..T.row_tiles) |mi| { + inline for (0..2) |h| { + var sum = results[T.stats + mi * 2 + h]; + inline for (.{ 2, 1 }) |offset| { + sum = sum.add(b.shuffleXor(sum, offset)); + } + const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); + const query = t.queryOf(mi, h); + var valid = b.openIf(query.lt(cfg.query_length)); + { + const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); + inline for (0..T.out_tiles) |di| { + const acc = unpackAcc(b, results[T.accIndex(mi, di)]); + const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); + b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); + } + valid.close(); + } + } + } +} + +test "CuTe attention emits tiled online softmax through head dimension 512" { + for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { + const ir = try Kernel.emit(std.testing.allocator, .{ + .batch = 2, + .query_length = 7, + .key_length = 37, + .query_heads = 6, + .kv_heads = 2, + .head_dim = head_dim, + .dtype = .bf16, + }, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.for") != null); + } +} diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig new file mode 100644 index 00000000..36e0953b --- /dev/null +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -0,0 +1,96 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const kernel = @import("attention.zig"); + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major >= 8; +} + +/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); + const batched = q.shape().hasTag(.b) != null; + const rank: usize = if (batched) 4 else 3; + stdx.debug.assert(q.rank() == rank and k.rank() == rank and v.rank() == rank, "CuTe attention expects rank-3 tensors or rank-4 tensors with a batch axis", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe attention requires matching Q/K/V dtypes", .{}); + stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); + stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); + stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); + stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); + stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); + if (batched) { + stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); + stdx.debug.assert(token_index.rank() == 0 or (token_index.rank() == 1 and token_index.shape().hasTag(.b) != null and token_index.dim(.b) == q.dim(.b)), "CuTe attention requires a scalar position or one position per batch element", .{}); + } else { + stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); + } + + const positions = if (batched) + token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) + else + token_index.convert(.i32); + const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); + const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); + const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); + + const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); + + const cfg: kernel.Config = .{ + .batch = if (batched) q_.dim(.b) else 1, + .query_length = q_.dim(.q), + .key_length = k_.dim(.k), + .query_heads = q_.dim(.h), + .kv_heads = k_.dim(.h), + .head_dim = q_.dim(.hd), + .dtype = zml.kernel.cute.from(q_.dtype()), + .splits = splits, + .causal = causal, + }; + + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, + .block = kernel.block, + }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; + + return out.transpose(q.shape()); +} + +/// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. +/// If there are no devices, returns 0 for both values. +/// If any device does not have the required attributes, returns 0 for both values. +fn decodeResources(platform: *const zml.Platform) kernel.DecodeResources { + var result: kernel.DecodeResources = .{ .multiprocessors = std.math.maxInt(i64), .shared_bytes = std.math.maxInt(i64) }; + if (platform.devices.len == 0) return .{ .multiprocessors = 0, .shared_bytes = 0 }; + for (platform.devices) |device| { + const sms = device.pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + const shared = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + result.multiprocessors = @min(result.multiprocessors, sms.int64); + result.shared_bytes = @min(result.shared_bytes, shared.int64); + } + return result; +} diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig new file mode 100644 index 00000000..3fb898db --- /dev/null +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -0,0 +1,139 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const cute = @import("cute/cute.zig"); +pub const metal = @import("metal/metal.zig"); +pub const nki = @import("nki/nki.zig"); +const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); + +test { + std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + return switch (platform.target) { + .neuron => true, + .metal => true, + .cuda => if (cute.isAvailable(platform)) true else false, + .cpu, .rocm, .tpu, .oneapi, .furiosa => false, + }; +} + +pub const Backend = enum { + cute, + nki, + metal, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .neuron => .nki, + .metal => .metal, + .cuda => .cute, + .cpu, .rocm, .tpu, .oneapi, .furiosa => stdx.debug.panic("Flash attention is not available on the {} platform", .{platform.target}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .cute => cute.isAvailable(platform), + .nki => platform.target == .neuron, + .metal => platform.target == .metal, + }; + } +}; + +pub const Parameters = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub const InitOptions = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = .init() }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => |v| .{ .nki = v }, + .metal => .{ .metal = {} }, + }; + } +}; + +pub const Metadata = union(Backend) { + cute: void, + nki: void, + metal: metal.Metadata, + + pub const InitOptions = union(Backend) { + cute: void, + nki: void, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .init() }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .cute => {}, + .nki => {}, + .metal => |*v| metal.Metadata.deinitBuffer(v), + } + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { + zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); + return switch (parameters) { + .cute => cute.attention(q, k, v, token_index, opts.causal), + // The NKI and Metal kernels are causal only. + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), + }; +} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel new file mode 100644 index 00000000..f82ade45 --- /dev/null +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "flash_attn.zig", + "metal.zig", + ], +) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig new file mode 100644 index 00000000..de4b68da --- /dev/null +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -0,0 +1,732 @@ +//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` +//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, +//! fa_tiled otherwise. + +const std = @import("std"); + +const metal = @import("../../../kernel/kernel.zig").metal; + +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVec = struct { + pub const Cfg = struct { + head_dim: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "k_len", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + k_len: i32, + scale: f32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec", + .inputs = &.{ "q", "k", "v", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k = char_in, + .v = char_in, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const n_kv = a.args.field("n_kv"); + const group = num_heads.div(n_kv); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + const kv_len = a.token_index.load(r).add(1); + const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const position_bytes = n_kv.to(.ulong).mul(D * 2); + const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); + const k0 = a.k.add(first); + const v0 = a.v.add(first); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, iwg); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first_row) |*scope| scope.close(); + } +}; + +fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { + return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); +} + +/// fa_vec_reduce: merges the fa_vec split partials. +pub const FaVecReduce = struct { + pub const Cfg = struct { + splits: i32, + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_reduce", + .inputs = &.{ "htmp", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVec.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + const live = liveSplits(b, a.token_index.load(r).add(1), NWG); + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past +/// `live` output zeros. +pub const FaTiled = struct { + pub const Cfg = struct { + dtype: Type, + head_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_tiled", + .inputs = &.{ "q", "k", "v", "token_index", "live" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, + .run = run, + }); + + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k = .{ .buffer = .{ .type = T, .const_ = true } }, + .v = .{ .buffer = .{ .type = T, .const_ = true } }, + .token_index = int_in, + .live = int_in, + .num_kv_heads = int_arg, + .scale = .{ .constant = .float }, + .q_len = int_arg, + .k_len = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_pos_start = a.tgp.y().to(.int).mul(BQ); + const batch = a.tgp.z().to(.int); + const num_heads = a.tgpg.x().to(.int); + const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); + const q_stride = num_heads.mul(HEAD_SIZE); + const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); + + const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); + const live = a.live.load(0); + const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); + const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + + var dead = b.openIf(valid_q.le(0)); + { + var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = zero.iv; + out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); + zero.close(); + } + b.return_(); + dead.close(); + } + + // Query i of the batch row is at position context_len + i. + const context_len = a.token_index.load(batch); + const seq_len = context_len.add(live); + const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + var past = b.openIf(tile_start.gt(last_q_pos)); + { + tiles.breakLoop(); + past.close(); + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); + const k_ptr = a.k.add(off); + const v_ptr = a.v.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj).mul(scale_log2); + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); + out_base.store(r.mul(q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the flash attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = FaTiled.tileConfig(head_size).?; + allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); + } + } +} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig new file mode 100644 index 00000000..bdfb00c0 --- /dev/null +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -0,0 +1,143 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const flash_attn = @import("flash_attn.zig"); + +pub const Metadata = struct { + /// Live queries of a single-sequence prefill; 0 means all. + num_tokens: zml.Tensor, + + pub fn init() Metadata { + return .{ .num_tokens = .fromShape(.scalar(.u32)) }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + _ = self; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.num_tokens.deinit(); + } +}; + +/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for +/// unsupported dtypes and head sizes. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { + const dtype = q.dtype(); + const head_dim = q.dim(.hd); + if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { + const seq_len = k.dim(.k); + const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); + return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + } + + const batched = q.shape().hasTag(.b) != null; + const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); + const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); + const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); + + const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; + const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) + decode(qb, kb, vb, positions, scale) + else + prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); + return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); +} + +fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { + const all: zml.Tensor = .scalar(q.dim(.q), .i32); + if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; + const num_tokens = metadata.num_tokens.convert(.i32); + return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); +} + +/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { + const batch: i32 = @intCast(q.dim(.b)); + const num_heads: i32 = @intCast(q.dim(.h)); + const head_dim: i32 = @intCast(q.dim(.hd)); + const num_kv_heads: i32 = @intCast(k.dim(.h)); + const keys: i32 = @intCast(k.dim(.k)); + const nrows = batch * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); + const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; + + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ + .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; +} + +/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { + const queries: i32 = @intCast(q.dim(.q)); + const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; + const cfg: flash_attn.FaTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + }; + return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), + .scale = .of(scale), + .q_len = .of(queries), + .k_len = .of(@as(i32, @intCast(k.dim(.k)))), + }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} + +test { + std.testing.refAllDecls(flash_attn); +} diff --git a/zml/kernels/flash_attention/nki/BUILD.bazel b/zml/kernels/flash_attention/nki/BUILD.bazel new file mode 100644 index 00000000..f8e4f470 --- /dev/null +++ b/zml/kernels/flash_attention/nki/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "attention_kernel", + srcs = ["attention.py"], +) + +filegroup( + name = "nki", + srcs = ["nki.zig"], +) + +py_binary( + name = "attention_harness", + srcs = ["attention_harness.py"], + data = [ + ":attention.py", + "@libzml_neuron//:sandbox", + ], + visibility = ["//visibility:public"], + deps = [ + "@neuron_py_deps//ml_dtypes:pkg", + "@neuron_py_deps//neuronx_cc:pkg", + "@neuron_py_deps//nki:pkg", + "@neuron_py_deps//numpy:pkg", + "@rules_python//python/runfiles", + ], +) diff --git a/zml/kernels/flash_attention/nki/attention.py b/zml/kernels/flash_attention/nki/attention.py new file mode 100644 index 00000000..cf76d9ae --- /dev/null +++ b/zml/kernels/flash_attention/nki/attention.py @@ -0,0 +1,473 @@ +import math + +import nki.isa as nisa +import nki.language as nl + +NEG_INF = -3.3895313892515355e38 + + +def decode(q, k, v, token_index): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q_len == 1 + assert k.shape == v.shape + assert q.shape[2] == k.shape[2] + assert num_q_heads % num_kv_heads == 0 + + VALUE_TILE_TOKENS = nl.tile_size.pmax + assert seq_len % VALUE_TILE_TOKENS == 0, ( + "seq_len " + f"({seq_len}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + num_kv_blocks = seq_len // BLOCK_K + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + token = nl.ndarray((1, 1), dtype=token_index.dtype, buffer=nl.sbuf) + nisa.dma_copy(dst=token, src=token_index[0:1, 0:1]) + + positions = nl.ndarray((q_len, BLOCK_K), dtype=token_index.dtype, buffer=nl.sbuf) + qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.psum) + qk_sbuf = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + masked_qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + neg_tile = nl.full( + (heads_per_kv, BLOCK_K), NEG_INF, dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_scale = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + neg_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + tile_sum = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + inverse_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.psum) + attn_out_sbuf = nl.ndarray((heads_per_kv, d_head), dtype=q.dtype, buffer=nl.sbuf) + + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=v.dtype, buffer=nl.sbuf + ) + v_sbuf = nl.ndarray((VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf) + + for kv_head in nl.affine_range(num_kv_heads): + q_head = kv_head * heads_per_kv + + q_sbuf = nl.ndarray((d_head, heads_per_kv), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[0, q_head : q_head + heads_per_kv, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + nisa.iota(positions, [[1, BLOCK_K]]) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(0, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=row_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_scalar( + dst=neg_row_max, + data=row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=sum_row, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + nisa.tensor_copy(dst=running_out, src=attn_out_psum) + + for k_tail_idx in nl.affine_range(num_kv_blocks - 1): + k_block_idx = k_tail_idx + 1 + k_start = k_block_idx * BLOCK_K + nisa.iota(positions, [[1, BLOCK_K]], offset=k_start) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(k_start, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=tile_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(k_start + value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + nisa.reciprocal(dst=inverse_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=running_out, + data=running_out, + op0=nl.multiply, + operand0=inverse_sum_row, + ) + nisa.tensor_copy(dst=attn_out_sbuf, src=running_out) + + nisa.dma_copy( + dst=out[0, q_head : q_head + heads_per_kv, :], + src=attn_out_sbuf, + ) + + return out + + +def prefill(q, k, v): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q.shape[2] == k.shape[2] + assert k.shape == v.shape + assert num_q_heads % num_kv_heads == 0 + assert q_len == seq_len + + VALUE_TILE_TOKENS = nl.tile_size.pmax + BLOCK_Q = nl.tile_size.pmax + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + + assert q_len % BLOCK_Q == 0, ( + f"q_len ({q_len}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % BLOCK_Q == 0, ( + f"BLOCK_K ({BLOCK_K}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + + num_q_blocks = q_len // BLOCK_Q + q_blocks_per_k_tile = BLOCK_K // BLOCK_Q + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + + for kv_head in nl.affine_range(num_kv_heads): + for q_idx in nl.affine_range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + + for q_block_idx in nl.affine_range(num_q_blocks): + q_start = q_block_idx * BLOCK_Q + + q_sbuf = nl.ndarray((d_head, BLOCK_Q), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[nl.ds(q_start, BLOCK_Q), q_head, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + + masked_qk = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + qk_sbuf = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + old_max_scale = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + neg_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_sum = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + new_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.psum + ) + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=v.dtype, buffer=nl.sbuf + ) + qk_psum = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.psum + ) + v_sbuf = nl.ndarray( + (VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf + ) + + nisa.memset(dst=row_max, value=NEG_INF) + nisa.memset(dst=sum_row, value=0.0) + nisa.memset(dst=running_out, value=0.0) + active_k_tiles = q_block_idx // q_blocks_per_k_tile + 1 + + for k_block_idx in nl.affine_range(active_k_tiles): + k_start = k_block_idx * BLOCK_K + k_sbuf = nl.ndarray( + (d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf + ) + nisa.dma_transpose( + dst=k_sbuf, + src=k[nl.ds(k_start, BLOCK_K), kv_head, :], + ) + nisa.memset(dst=qk_psum, value=0.0) + nisa.nc_matmul(dst=qk_psum, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk_psum) + nisa.affine_select( + dst=masked_qk, + pattern=[[-1, BLOCK_K]], + offset=q_start - k_start, + channel_multiplier=1, + on_true_tile=qk_sbuf, + on_false_value=NEG_INF, + cmp_op=nl.greater_equal, + ) + nisa.tensor_reduce( + dst=tile_max, + op=nl.maximum, + data=masked_qk, + axis=(1,), + ) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + nisa.dma_copy( + dst=v_sbuf, + src=v[ + nl.ds(k_start + value_start, VALUE_TILE_TOKENS), + kv_head, + :, + ], + ) + nisa.nc_matmul( + dst=attn_out_psum, + stationary=prob_t, + moving=v_sbuf, + ) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + inv_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + normalized_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out = nl.ndarray((BLOCK_Q, d_head), dtype=q.dtype, buffer=nl.sbuf) + nisa.reciprocal(dst=inv_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=normalized_out, + data=running_out, + op0=nl.multiply, + operand0=inv_sum_row, + ) + nisa.tensor_copy(dst=attn_out, src=normalized_out) + + nisa.dma_copy( + dst=out[nl.ds(q_start, BLOCK_Q), q_head, :], + src=attn_out, + ) + + return out diff --git a/zml/kernels/flash_attention/nki/attention_harness.py b/zml/kernels/flash_attention/nki/attention_harness.py new file mode 100644 index 00000000..34ce5717 --- /dev/null +++ b/zml/kernels/flash_attention/nki/attention_harness.py @@ -0,0 +1,331 @@ +import argparse +import dataclasses +import importlib.util +import os +import sys +import time +from pathlib import Path + +import ml_dtypes +import nki +import numpy as np +from python.runfiles import Runfiles + + +DEFAULT_SEQ_LEN = 128 +ATOL = 1e-2 +RTOL = 1e-2 +MIN_CLOSE_FRACTION = 0.99 +INPUT_CASES = ("baseline", "signed") + + +@dataclasses.dataclass(frozen=True) +class ModelSpec: + name: str + num_heads: int + num_kv_heads: int + head_dim: int + + @property + def heads_per_kv(self): + return self.num_heads // self.num_kv_heads + + +MODEL_PRESETS = { + "llama-3.1-8b": ModelSpec( + name="llama-3.1-8b", + num_heads=32, + num_kv_heads=8, + head_dim=128, + ), +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run the classic Neuron NKI attention correctness contract." + ) + parser.add_argument( + "--model", + choices=(*MODEL_PRESETS.keys(), "custom"), + default="llama-3.1-8b", + ) + parser.add_argument("--seq-len", type=int, default=DEFAULT_SEQ_LEN) + parser.add_argument("--dtype", choices=("bf16", "f16", "f32"), default="bf16") + parser.add_argument("--run", choices=("both", "prefill", "decode"), default="both") + parser.add_argument("--mode", choices=("verify", "benchmark"), default="verify") + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--num-heads", type=int, default=None) + parser.add_argument("--num-kv-heads", type=int, default=None) + parser.add_argument("--head-dim", type=int, default=None) + return parser.parse_args() + + +def model_spec(args): + custom_values = (args.num_heads, args.num_kv_heads, args.head_dim) + if args.model != "custom": + if any(value is not None for value in custom_values): + raise ValueError("shape flags are only accepted with --model=custom") + return MODEL_PRESETS[args.model] + + if any(value is None for value in custom_values): + raise ValueError("--model=custom requires --num-heads, --num-kv-heads, and --head-dim") + spec = ModelSpec( + name="custom", + num_heads=args.num_heads, + num_kv_heads=args.num_kv_heads, + head_dim=args.head_dim, + ) + validate_spec(spec) + return spec + + +def validate_spec(spec): + if spec.num_heads <= 0 or spec.num_kv_heads <= 0 or spec.head_dim <= 0: + raise ValueError("model dimensions must be positive") + if spec.num_heads % spec.num_kv_heads != 0: + raise ValueError("num_heads must be divisible by num_kv_heads") + + +def validate_seq_len(seq_len): + if seq_len <= 0 or seq_len % 128 != 0: + raise ValueError("--seq-len must be a positive multiple of 128") + + +def validate_execution_args(args): + if args.mode == "verify": + return + if args.warmups < 0: + raise ValueError("--warmups must be non-negative") + if args.iterations <= 0: + raise ValueError("--iterations must be positive") + + +def decode_positions(seq_len): + return tuple(dict.fromkeys((0, 1, seq_len // 2, seq_len - 1))) + + +def configure_runtime_environment(mode): + if mode == "benchmark": + os.environ.pop("NEURON_RT_LOG_LEVEL", None) + + +def load_nki_attention_module(): + path = Path(__file__).with_name("attention.py") + spec = importlib.util.spec_from_file_location("zml_nki_attention", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def add_neuron_tools_to_path(): + runfiles = Runfiles.Create() + python_neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/python-shims/neuronx-cc") + ) + neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/neuronx-cc") + ) + os.environ["PATH"] = os.pathsep.join( + (str(python_neuronx_cc.parent), str(neuronx_cc.parent), os.environ["PATH"]) + ) + + +def numpy_dtype(name): + if name == "bf16": + return ml_dtypes.bfloat16 + if name == "f16": + return np.float16 + if name == "f32": + return np.float32 + raise ValueError(f"unsupported dtype: {name}") + + +def fill_input(shape, offset, input_case, dtype): + values = np.linspace(0.0, 1.0, num=int(np.prod(shape)), endpoint=False, dtype=np.float32) + if input_case == "signed": + data = offset + (values - 0.5) * 0.25 + else: + data = offset + values * 0.03 + return data.reshape(shape).astype(dtype) + + +def qkv_inputs(spec, seq_len, input_case, decode, dtype): + q_len = 1 if decode else seq_len + q = fill_input((q_len, spec.num_heads, spec.head_dim), 0.01, input_case, dtype) + k = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.02, input_case, dtype) + v = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.03, input_case, dtype) + return q, k, v + + +def reference_attention(q, k, v, token_index=None): + q_len, num_heads, head_dim = q.shape + seq_len, num_kv_heads, _ = k.shape + heads_per_kv = num_heads // num_kv_heads + scale = 1.0 / np.sqrt(head_dim) + q_f32 = q.astype(np.float32) + k_f32 = k.astype(np.float32) + v_f32 = v.astype(np.float32) + out = np.empty((q_len, num_heads, head_dim), dtype=np.float32) + + if token_index is None: + valid = np.arange(seq_len)[None, :] <= np.arange(q_len)[:, None] + else: + if q_len != 1: + raise ValueError("decode reference expects q_len=1") + valid = (np.arange(seq_len) <= token_index)[None, :] + + for kv_head in range(num_kv_heads): + keys = k_f32[:, kv_head, :] + values = v_f32[:, kv_head, :] + for q_idx in range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + scores = (q_f32[:, q_head, :] @ keys.T) * scale + scores = np.where(valid, scores, -np.inf) + scores -= np.max(scores, axis=1, keepdims=True) + probs = np.exp(scores) + probs /= np.sum(probs, axis=1, keepdims=True) + out[:, q_head, :] = probs @ values + + return out.astype(q.dtype) + + +def assert_attention_close(label, expected, actual): + expected_f32 = expected.astype(np.float32) + actual_f32 = actual.astype(np.float32) + finite = np.isfinite(actual_f32) + close = np.isclose(expected_f32, actual_f32, atol=ATOL, rtol=RTOL) + close_fraction = float(close.mean()) + max_abs = float(np.nanmax(np.abs(expected_f32 - actual_f32))) + print( + f"{label} finite={int(finite.sum())}/{actual_f32.size} " + f"close_fraction={close_fraction:.6f} max_abs={max_abs:.6g}" + ) + if not finite.all(): + raise AssertionError(f"{label}: output contains NaN or Inf") + if close_fraction < MIN_CLOSE_FRACTION: + raise AssertionError(f"{label}: close_fraction={close_fraction:.6f}") + + +def timed_run(kernel, inputs, mode, warmups, iterations): + if mode == "verify": + start = time.perf_counter() + output = kernel(*inputs) + elapsed = time.perf_counter() - start + return output, elapsed, 1 + + output = kernel(*inputs) + for _ in range(warmups): + output = kernel(*inputs) + + total = 0.0 + for _ in range(iterations): + start = time.perf_counter() + output = kernel(*inputs) + total += time.perf_counter() - start + return output, total, iterations + + +def print_timing(label, elapsed, iterations, output): + if iterations == 1: + print(f"{label} elapsed={elapsed * 1000.0:.3f}ms output={output.shape}") + return + + average = elapsed / iterations + print( + f"{label} total={elapsed * 1000.0:.3f}ms " + f"avg={average * 1000.0:.3f}ms iterations={iterations} output={output.shape}" + ) + + +def run_prefill(prefill_kernel, spec, seq_len, input_case, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=False, dtype=dtype) + output, elapsed, timed_iterations = timed_run( + prefill_kernel, + (q, k, v), + mode, + warmups, + iterations, + ) + label = f"prefill input_case={input_case}" + if mode == "verify": + expected = reference_attention(q, k, v) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def run_decode(decode_kernel, spec, seq_len, input_case, token_index, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=True, dtype=dtype) + token = np.array([[token_index]], dtype=np.uint32) + output, elapsed, timed_iterations = timed_run( + decode_kernel, + (q, k, v, token), + mode, + warmups, + iterations, + ) + label = f"decode input_case={input_case} token_index={token_index}" + if mode == "verify": + expected = reference_attention(q, k, v, token_index) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def main(): + args = parse_args() + spec = model_spec(args) + validate_spec(spec) + validate_seq_len(args.seq_len) + validate_execution_args(args) + positions = decode_positions(args.seq_len) + dtype = numpy_dtype(args.dtype) + + configure_runtime_environment(args.mode) + add_neuron_tools_to_path() + module = load_nki_attention_module() + prefill_kernel = nki.jit(module.prefill) + decode_kernel = nki.jit(module.decode) + + header = ( + "classic_attention_contract " + f"model={spec.name} seq_len={args.seq_len} num_heads={spec.num_heads} " + f"num_kv_heads={spec.num_kv_heads} heads_per_kv={spec.heads_per_kv} " + f"head_dim={spec.head_dim} dtype={args.dtype} run={args.run} " + f"mode={args.mode}" + ) + if args.mode == "benchmark": + header += f" warmups={args.warmups} iterations={args.iterations}" + print(f"{header} decode_positions={positions}") + for input_case in INPUT_CASES: + if args.run in ("both", "prefill"): + run_prefill( + prefill_kernel, + spec, + args.seq_len, + input_case, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + if args.run in ("both", "decode"): + for token_index in positions: + run_decode( + decode_kernel, + spec, + args.seq_len, + input_case, + token_index, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + + +if __name__ == "__main__": + main() diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig new file mode 100644 index 00000000..f98a545d --- /dev/null +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -0,0 +1,59 @@ +const platforms = @import("platforms"); + +const zml = @import("../../../zml.zig"); +const stdx = zml.stdx; + +const KernelSpec = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, +}; + +pub const Parameters = struct { + compiler_target: []const u8, + decode_kernel: KernelSpec, + prefill_kernel: KernelSpec, + + pub fn init() Parameters { + if (comptime platforms.isEnabled(.neuron)) { + const nki_kernel = @import("platforms/neuron/nki_kernel"); + const compiler_target = nki_kernel.compilerTargetFromInstance(); + return .{ + .compiler_target = @tagName(compiler_target), + .decode_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_decode", + .entrypoint = "decode", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + .prefill_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_prefill", + .entrypoint = "prefill", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + }; + } + + stdx.debug.panic("NKI attention requires the Neuron platform", .{}); + } +}; + +/// Attention for Neuron. Both prefill and decode lower to NKI custom calls. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { + const decode = q.dim(.q) == 1; + const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; + const options: zml.ops.NeuronNkiOps = .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = parameters.compiler_target, + }; + const output = if (decode) + zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] + else + zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; + return output.convert(q.dtype()); +} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..c3c3dca2 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,346 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { + try testAttention( + .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), + .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), + &.{3000}, + ); +} + +test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), + .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), + &.{900}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig new file mode 100644 index 00000000..b66593c1 --- /dev/null +++ b/zml/kernels/moe/api.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_8 => blk: { + const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); + break :blk .{ + grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + }; + }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { + concatenated, // G0 G1 … Gn U0 U1 … Un + row_interleaved_1, // G0 U0 G1 U1 … + row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … +}; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Gate/up ordering before applying the backend-specific packing. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte + /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, + /// represented as `[expert, k/128, rows]` i32 words of four scales. + mxfp4_interleaved8, + }; +}; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel new file mode 100644 index 00000000..ca87f37a --- /dev/null +++ b/zml/kernels/moe/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "//zml/kernels/moe/cute/sm10x", + ], +) diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig new file mode 100644 index 00000000..8bc03c9c --- /dev/null +++ b/zml/kernels/moe/cute/cute.zig @@ -0,0 +1,139 @@ +//! Blackwell MXFP4 and NVFP4 MoE backend with persistent CuTe GEMMs. + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; +pub const kernels = @import("sm10x/moe.zig"); + +// `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the +// kernel emit tests are only collected if `kernels` is referenced here. +test { + _ = kernels; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +/// The NVFP4 kernels are only validated on SM103. +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10 and cc.minor == 3; +} + +pub fn validateOptions(opts: Options) void { + stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); +} + +/// MXFP4 weights are `u8` (or `i8`) packed FP4 with E8M0 block scales. NVFP4 +/// weights are packed `u8` with E4M3 block scales; their weight global scale +/// is the per-expert multiplier dequantizing each GEMM output (it folds in +/// the inverse of the input's global scale), and their input scale is the +/// multiplier applied to the GEMM's input before NVFP4 quantization. Both +/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows +/// interleave by 8 and its scale atoms are MN-major packed i32 words. +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute backend expects gate_up bias and down bias to be null", .{}); + const gq = gate_up.quantization orelse @panic("cute backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("cute backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == dq.scheme, "cute expects gate_up and down quantization scheme to match, got {} and {}", .{ gq.scheme, dq.scheme }); + + const format: Format = switch (gq.scheme) { + .mxfp4 => blk: { + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{down.weight.dtype()}); + // Check the boundary quantizer and CuTe GEMM tile constraints. + kernels.validateMxfp4Shapes(down.weight.dim(1), down.weight.dim(2) * 2); + // The SwiGLU clamp is a kernel parameter, but the routing weight is + // multiplied in the up epilogue, before the down projection: applying it + // after would have to move into the down epilogue or the reduction. + stdx.debug.assert(options.routing_weight_placement == .before_down, "cute MXFP4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); + break :blk .mxfp4; + }, + .nvfp4 => blk: { + stdx.debug.assert(!options.quantize_input, "cute NVFP4 backend always quantizes its inputs to NVFP4", .{}); + stdx.debug.assert(gq.swizzled_scales and dq.swizzled_scales, "cute NVFP4 backend expects swizzled block scales", .{}); + stdx.debug.assert(gate_up.weight.dtype() == .u8 and down.weight.dtype() == .u8, "cute NVFP4 backend expects packed u8 weights, got {} and {}", .{ gate_up.weight.dtype(), down.weight.dtype() }); + const scale = struct { + fn f(global: ?zml.Quantization.GlobalScale) zml.Tensor { + return (global orelse @panic("cute NVFP4 backend requires global and input scales")).asMultiplier(); + } + }.f; + break :blk .{ .nvfp4 = .{ + .alpha1 = scale(gq.global_scale), + .alpha2 = scale(dq.global_scale), + .act1 = scale(gq.input_scale), + .act2 = scale(dq.input_scale), + } }; + }, + else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), + }; + + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + return switch (format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .ids = ids, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + }), + }.reshape(input.shape().dims()).withTags(input.shape()); +} + +/// The format-specific tensors: NVFP4's per-expert global scales. They stay +/// as given until the sharded body, so they split across ranks like the +/// weights. +const Format = union(enum) { + mxfp4, + nvfp4: struct { + alpha1: zml.Tensor, + alpha2: zml.Tensor, + act1: zml.Tensor, + act2: zml.Tensor, + }, +}; diff --git a/zml/kernels/moe/cute/sm10x/BUILD.bazel b/zml/kernels/moe/cute/sm10x/BUILD.bazel new file mode 100644 index 00000000..2b23bfbb --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/BUILD.bazel @@ -0,0 +1,14 @@ +package(default_visibility = ["//zml/kernels/moe/cute:__pkg__"]) + +filegroup( + name = "sm10x", + srcs = [ + "common.zig", + "moe.zig", + "mxfp4.zig", + "nvfp4.zig", + "persistent_mxfp4.zig", + "persistent_nvfp4.zig", + "reduce.zig", + ], +) diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig new file mode 100644 index 00000000..da639c75 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -0,0 +1,227 @@ +//! Machinery shared by the MXFP4 and NVFP4 CuTe MoE kernels: pipeline +//! barriers and state, the static persistent tile scheduler, direct routing, +//! shared-memory pointers, boundary kernel launches and small IR helpers. +const std = @import("std"); +const mlir = @import("mlir"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_ } }; +} + +pub fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; +} + +pub fn barrierAt(b: *B, base: V, stage: V) V { + return b.addOffset( + base, + b.makeIntTuple(.{stage}), + b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), + ).value(); +} + +pub const PipelinePosition = struct { + index: V, + phase: V, +}; + +/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the +/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L +/// coordinate, validity predicate, linear work index, and executed-tile +/// count through every role's `scf.while`. Keeping the same state shape is +/// important to the CuTe software-pipeline passes, even though N is always +/// zero for one tile per group. +pub const PersistentWork = struct { + m: V, + n: V, + group: V, + valid: V, + linear: V, + executed: V, +}; + +/// Direct routing (one token) runs one single-row group per route. Routes +/// owned by other expert-parallel ranks carry expert -1: the persistent tiles +/// cover the valid routes only, in route order, so they do not occupy a wave. +pub const DirectRoutes = struct { + expert_ids: cute.View, + routes: usize, + + pub fn count(self: DirectRoutes, b: *B) V { + var valid = b.constant(.i32, 0); + for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); + return valid; + } + + /// Route of the `k`-th valid route. + fn nth(self: DirectRoutes, b: *B, k: V) V { + var seen = b.constant(.i32, 0); + var route = b.constant(.i32, 0); + for (0..self.routes) |r| { + const valid = self.expertAt(b, r).ge(0); + route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); + seen = seen.add(valid.to(.i32)); + } + return route; + } + + fn expertAt(self: DirectRoutes, b: *B, r: usize) V { + return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); + } +}; + +pub fn persistentWorkAt( + b: *B, + linear: V, + executed: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + // StaticPersistentTileScheduler uses two first-class FastDivmod objects + // to decode the column-major (M, N=1, L) problem layout. Preserve those + // operations even though ordinary rem/div is mathematically equivalent; + // CuTe's scheduler reconstruction and loop canonicalization key off these + // values and carry them through every persistent loop. + const group_and_m = b.fastDivmod(linear, div_m); + const group_and_n = b.fastDivmod(group_and_m[0], div_n); + const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; + return .{ + .m = b.makeWarpUniform(group_and_m[1]), + .n = b.makeWarpUniform(group_and_n[1]), + .group = b.makeWarpUniform(group), + .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), + .linear = linear, + .executed = executed, + }; +} + +pub fn advancePersistentWork( + b: *B, + linear: V, + executed: V, + persistent_clusters: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); +} + +/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase +/// as explicit loop-carried SSA values instead of recovering them with +/// division/remainder; the generated CuTe program then has the same circular +/// state transition as the Python DSL. +pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { + const advanced = index.add(1); + const wraps = advanced.eq(stages); + const next_index = b.select(wraps, 0, advanced); + var next_phase = b.openIfElse(wraps, .{phase}); + { + next_phase.yieldThen(.{phase.bitXor(1)}); + } + { + next_phase.yieldElse(.{phase}); + } + return .{ .index = next_index, .phase = next_phase.results[0] }; +} + +pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { + var peek = b.openIfElse(count.lt(limit), .{.i1}); + { + peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); + } + { + peek.yieldElse(.{b.constant(.i1, true)}); + } + return peek.results[0]; +} + +/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the +/// base alignment and the offset, and `cute.recast_iter` may not change it, so +/// the field's own alignment has to be what the offset leaves. +pub fn sharedPointerAt( + b: *B, + base: V, + comptime byte_offset: i64, + comptime dtype: cute.DType, + comptime alignment: u64, +) V { + const moved = comptime movedAlignment(byte_offset); + comptime std.debug.assert(moved >= alignment); + const byte_ptr = b.addOffset( + base, + b.makeIntTuple(.{byte_offset}), + b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), + ); + return b.recastPointer(byte_ptr, dtype, .smem, moved); +} + +/// Alignment of `shared_base + byte_offset`, which the shared storage is +/// allocated with. +fn movedAlignment(comptime byte_offset: i64) u64 { + return offsetAlignment(1024, byte_offset, 1); +} + +/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, +/// and the op's result type has to say so. +pub fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { + if (offset_elements == 0) return base_bytes; + const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; + return @max(std.math.gcd(base_bytes, moved), element_bytes); +} + +pub fn Tuple(comptime T: type, comptime len: usize) type { + return @Tuple(&@as([len]type, @splat(T))); +} + +pub fn arity(comptime T: type) usize { + return @typeInfo(T).@"struct".field_names.len; +} + +/// The MLIR types of a tuple of values, as `openWhile` expects them. +pub fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { + const len = comptime arity(@TypeOf(values)); + var out: Tuple(*const mlir.Type, len) = undefined; + inline for (0..len) |i| out[i] = values[i].type_(); + return out; +} + +/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. +pub fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { + var out: Tuple(@TypeOf(array[0]), array.len) = undefined; + inline for (0..array.len) |i| out[i] = array[i]; + return out; +} + +pub fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { + return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); +} + +pub fn view(b: *B, value: V) cute.View { + return .{ .inner = value.inner, .kernel = b }; +} + +/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. +pub fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { + const one = b.constant(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize(kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(kernel_name, config, args); + b.returnHostStatus(b.cudaResultStatus(launch)); +} diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig new file mode 100644 index 00000000..ff6edec2 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -0,0 +1,348 @@ +//! SM100 CuTe MoE, MXFP8 x MXFP4 (the Zig port of the CuTe DSL `NativeMoE`) +//! and NVFP4 x NVFP4: routing, routed input quantization, persistent up GEMM +//! with a fused SwiGLU + quantization epilogue, persistent down GEMM storing +//! rows in route order, and the top-k reduction. MXFP4 applies routing weights +//! before the down projection, with the reference model's BF16 rounding. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); +pub const mxfp4 = @import("mxfp4.zig"); +pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); +pub const nvfp4 = @import("nvfp4.zig"); +pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); +const reduce = @import("reduce.zig"); + +pub const Mxfp4Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. + w1: zml.Tensor, + /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, + /// in MN-major 128x4 atoms, with rows interleaved like `w1`. + s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. + w2: zml.Tensor, + /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. + s2: zml.Tensor, + /// `[tokens, topk]` local expert ids. + ids: zml.Tensor, +}; + +/// The boundary quantizer handles four 128-column tiles per CTA. The GEMMs +/// support 128-column tiles and a fixed set of reduction tile counts. +pub fn validateMxfp4Shapes(hidden: i64, intermediate: i64) void { + stdx.debug.assert(hidden > 0 and @rem(hidden, 4 * 128) == 0, "cute backend requires hidden dimension to be a positive multiple of 512, got {}", .{hidden}); + stdx.debug.assert(intermediate > 0 and @rem(intermediate, 128) == 0, "cute backend requires intermediate dimension to be a positive multiple of 128, got {}", .{intermediate}); + stdx.debug.assert(supportedReductionTiles(@divTrunc(hidden, 128)), "cute backend does not support hidden reduction tile count {}, got hidden dimension {}", .{ @divTrunc(hidden, 128), hidden }); + stdx.debug.assert(supportedReductionTiles(@divTrunc(intermediate, 128)), "cute backend does not support intermediate reduction tile count {}, got intermediate dimension {}", .{ @divTrunc(intermediate, 128), intermediate }); +} + +fn supportedReductionTiles(tiles: i64) bool { + return switch (tiles) { + 16, 18, 32, 40 => true, + else => false, + }; +} + +/// Route each token/top-k pair to its own one-row group instead of grouping +/// routes by expert. A token's top-k experts are distinct, so for a single +/// token grouping cannot share any weight tile and only adds the routing +/// kernels. Python `grouped=False`. +fn isDirect(tokens: i64) bool { + return tokens == 1; +} + +/// GEMM tile N, which is also the routed expert-group size. Wider tiles +/// reuse each weight tile across more routes but pad more rows per expert, +/// so they only pay off once experts receive tens of routes. +fn tileN(tokens: i64, experts: i64, topk: i64) i64 { + // Direct routing keeps one row per group: the narrowest MMA wastes the + // least of each B tile and leaves room for a 12-stage A/B ring. + if (isDirect(tokens)) return 8; + if (tokens < 512) return 16; + // Prefill: each group re-reads the whole expert weight matrix, so widen + // the tile once experts hold enough routes to fill it. Half the routes + // are remote under expert parallelism, hence the conservative estimate. + // The thresholds are measured on the DeepSeek V4.1 expert geometry. + const routes_per_expert = @divTrunc(tokens * topk, 2 * experts); + if (routes_per_expert >= 128) return 128; + if (routes_per_expert >= 32) return 64; + return 32; +} + +/// One persistent CTA per SM. Each CTA needs the full 227 KiB of shared +/// memory, so launching more CTAs than SMs (e.g. 160 on a 152-SM GB300) +/// runs the surplus as a second wave and nearly doubles the GEMM time. +fn persistentCtas() i64 { + const platform = zml.Compiler.current().platform; + if (platform.devices.len == 0) return 148; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 148; + return if (attribute.int64 > 0) attribute.int64 else 148; +} + +// TODO(Corentin): settle on groups/capacity. + +/// Upper bound on the number of `group`-row expert groups for all routes. +fn computeCapacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { + return @divTrunc(tokens * topk + group - 1, group) + experts - 1; +} + +/// Where each route lives in the grouped GEMM operands. +const Routing = struct { + /// Expert groups, i.e. the GEMMs' L extent. + groups: i64, + /// Route -> physical routed row. Null for direct routing, where route + /// `r` is row `r`. + route_map: ?zml.Tensor, + /// Physical routed row -> route. Direct routing does not read it. + route_inverse: zml.Tensor, + /// `[group experts | group sizes | active group count]` for grouped + /// routing, `[route experts]` for direct routing. + schedule: zml.Tensor, +}; + +const atomic_route_threshold = 6 * 2048; + +fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { + const routes = tokens * topk; + const flat_ids = ids.convert(.i32).reshape(.{routes}); + if (direct) return .{ .groups = routes, .route_map = null, .route_inverse = flat_ids, .schedule = flat_ids }; + const capacity = computeCapacity(tokens, topk, experts, n); + + std.debug.assert(capacity > 0); + // `route` rescans every route once per expert: cheapest for decode, but + // quadratic. Past a few thousand tokens, count with atomics instead. + const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { + const counted = triton_kernels.RouteAtomic.Kernel.call( + .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{experts}, .i32)) }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + }, + .grid = .{ @intCast(@divTrunc(routes + triton_kernels.RouteAtomic.route_block - 1, triton_kernels.RouteAtomic.route_block)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + .output_operand_aliases = .{ .counts = .zeros }, + }, + ); + break :blk .{ .counts = counted.counts, .pos = counted.pos }; + } else blk: { + const scanned = triton_kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ .tokens = tokens, .topk = topk, .expert_offset = 0 }, + .grid = .{ @intCast(experts), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; + }; + + const s = triton_kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ + .map = .init(.{capacity * n}, .i32), + .perm = .init(.{tokens * topk}, .i32), + .sched = .init(.{2 * capacity + 1}, .i32), + }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + .capacity = capacity, + .group = n, + }, + .grid = .{ @intCast(@divTrunc(tokens * topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + return .{ .groups = capacity, .route_map = s.perm, .route_inverse = s.map, .schedule = s.sched }; +} + +/// `[tokens, hidden]` BF16 output of the MXFP4 MoE layer. `tile_n` is the +/// GEMM tile N, for tuning; null selects it from the batch size. +pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Mxfp4Inputs, tile_n: ?i64) zml.Tensor { + const n = tile_n orelse tileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routing = route(tokens, experts, topk, n, direct, a.ids); + const routes = tokens * topk; + + const up_cfg: mxfp4_persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_mxfp8, + .routes = routes, + .direct = direct, + .swiglu_limit = swiglu_limit, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + + const q1 = mxfp4.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = mxfp4_persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1, + .input_scale = q1.s, + }, routing.route_inverse, a.routing_weights.reshape(.{routes})); + const down = mxfp4_persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2, + .input_scale = q2.s, + }, routing.route_inverse); + return reduce.topk(down, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), + }); +} + +pub const Nvfp4Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + /// `[tokens, topk]` local expert ids, -1 for routes of other ranks. + ids: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved. + w1: zml.Tensor, + /// `w1` E4M3 scales `[experts, 2 * intermediate, hidden / 16]` in the 128x4 layout. + s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. + w2: zml.Tensor, + /// `w2` E4M3 scales `[experts, hidden, intermediate / 16]` in the 128x4 layout. + s2: zml.Tensor, + /// `[experts]` FP32 dequantization scale of each GEMM output, which folds + /// in the inverse of the GEMM input's global scale. + alpha1: zml.Tensor, + alpha2: zml.Tensor, + /// `[experts]` FP32 global scale each GEMM's input is multiplied by + /// before NVFP4 quantization. + act1: zml.Tensor, + act2: zml.Tensor, +}; + +pub const Nvfp4Options = struct { + swiglu_limit: ?f32 = null, + routing_weight_placement: RoutingWeightPlacement = .after_down, + /// GEMM tile N; null selects it from the batch size. + tile_n: ?i64 = null, +}; + +pub fn validateNvfp4Shapes(hidden: i64, intermediate: i64) void { + const k_tile = nvfp4_persistent.k_tile; + stdx.debug.assert(@rem(hidden, k_tile) == 0, "cute NVFP4 backend expects hidden to be a multiple of {}, got {}", .{ k_tile, hidden }); + stdx.debug.assert(@rem(intermediate, k_tile) == 0, "cute NVFP4 backend expects intermediate to be a multiple of {}, got {}", .{ k_tile, intermediate }); +} + +/// NVFP4 GEMM tile N: the expected routes of one expert, so that each +/// expert's weight tiles are read once per group instead of once per few rows. +fn nvfp4TileN(tokens: i64, experts: i64, topk: i64) i64 { + if (isDirect(tokens)) return 8; + const per_expert = std.math.divCeil(i64, tokens * topk, experts) catch unreachable; + return @intCast(std.math.clamp(std.math.ceilPowerOfTwoAssert(u64, @intCast(per_expert)), 16, 128)); +} + +/// `[tokens, hidden]` BF16 output of the NVFP4 MoE layer. +pub fn forwardNvfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, a: Nvfp4Inputs, opts: Nvfp4Options) zml.Tensor { + validateNvfp4Shapes(hidden, intermediate); + const n = opts.tile_n orelse nvfp4TileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routes = tokens * topk; + const flat_ids = a.ids.convert(.i32).reshape(.{routes}); + const routing = route(tokens, experts, topk, n, direct, a.ids); + const route_map = routing.route_map orelse flat_ids; + const weights = a.routing_weights.convert(.f32).reshape(.{routes}); + + const up_cfg: nvfp4_persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_nvfp4, + .routes = routes, + .direct = direct, + .swiglu_limit = opts.swiglu_limit, + .routing_weights = opts.routing_weight_placement == .before_down, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + down_cfg.routing_weights = opts.routing_weight_placement == .after_down; + + const q1 = nvfp4.groupedQuantize(a.x, route_map, flat_ids, a.act1, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .experts = experts, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = nvfp4_persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1.bitCast(.u8), + .input_scale = q1.s, + .alpha = a.alpha1, + }, routing.route_inverse, weights, a.act2); + const down = nvfp4_persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2.bitCast(.u8), + .input_scale = q2.s, + .alpha = a.alpha2, + }, routing.route_inverse, weights); + return reduce.topk(down, route_map, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), + }); +} + +test { + _ = mxfp4; + _ = mxfp4_persistent; + _ = nvfp4; + _ = nvfp4_persistent; + _ = reduce; +} diff --git a/zml/kernels/moe/cute/sm10x/mxfp4.zig b/zml/kernels/moe/cute/sm10x/mxfp4.zig new file mode 100644 index 00000000..29e68977 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/mxfp4.zig @@ -0,0 +1,139 @@ +//! CuTe input quantization kernel in front of the persistent MXFP4 GEMMs of +//! `persistent_mxfp4.zig`: routed MXFP8 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute: it then starts while the previous +// kernel drains, and waits on `griddepcontrol` before its first read. A bare +// kernel is launched by XLA without that attribute. +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "mxfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; + +/// Quantize each token once, then scatter it to the physical rows assigned by +/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 +/// block-scaled MMA scale-factor tensor map. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +/// Python `prepare_input_vector`: each thread quantizes four consecutive +/// values and stores them as one packed word, so eight lanes cover one +/// 32-value MX block. +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + // Host parameters carry the custom call's element types; the kernel reads + // four FP8 values per 32-bit word. + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + }); + try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ + host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.endFunction(null); +} + +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + // Four FP8 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // This grid may have started before its producer finished: the routing and + // the previous layer write what it reads below. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(128).add(tid); + var x: [4]cute.Value = undefined; + for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); + + var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); + inline for (.{ 1, 2, 4 }) |offset| { + maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + } + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); + const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); + + const scale_col = word.div(8); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); + a.q.set(.{ row, word }, packed_word); + + var leader = b.openIf(tid.rem(8).eq(0)); + const scale_row = row.div(cfg.group_size); + const row_in_group = row.rem(cfg.group_size); + const swizzled_col = scale_col.div(4).mul(512) + .add(row_in_group.rem(32).mul(16)) + .add(row_in_group.div(32).mul(4)) + .add(scale_col.rem(4)); + a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); + leader.close(); + if (scheduled) |*s| s.close(); + } + // Only now are the rows this CTA owns complete: release the persistent + // GEMM, which waits on this signal before its first TMA read. + b.launchDependents(); +} + +test "MXFP4 CuTe input quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + }); + defer std.testing.allocator.free(quant_ir); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/sm10x/nvfp4.zig b/zml/kernels/moe/cute/sm10x/nvfp4.zig new file mode 100644 index 00000000..5afee090 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/nvfp4.zig @@ -0,0 +1,176 @@ +//! CuTe input quantization kernel in front of the persistent NVFP4 GEMMs of +//! `persistent_nvfp4.zig`: routed NVFP4 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + experts: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "nvfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map", "ids", "global_scale" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "nvfp4_cute_grouped_quantize_kernel"; +/// Packed FP4 words (eight values) a CTA quantizes per token. +const words_per_cta = 128; + +/// NVFP4 E4M3 block scale of a block maximum: `scale = e4m3(global * +/// maximum / 6)` and the quantization multiplier `global / scale`, zero for an +/// all-zero block. +pub fn blockScale(b: *B, maximum: V, global: V) struct { bits: V, multiplier: V } { + const raw = global.mul(maximum.mul(1.0 / 6.0)).minimum(448.0); + const zero = b.constant(.f32, 0); + const bits = b.packFp8x4(raw, zero, zero, zero).bitAnd(0xff); + const scale = b.e4m3ToF32(bits); + const inverse = b.unaryF32("rcp.approx.ftz.f32", scale.mul(b.unaryF32("rcp.approx.ftz.f32", global))); + return .{ .bits = bits, .multiplier = b.select(scale.ne(0.0), inverse, zero) }; +} + +/// Offset of the scale of row `row` (within its 128-row group) and 16-wide +/// block `block` in the 128x4 tensor-core layout. +pub fn swizzledScaleOffset(row: V, block: V) V { + return block.div(4).mul(512) + .add(row.rem(32).mul(16)) + .add(row.div(32).mul(4)) + .add(block.rem(4)); +} + +/// Quantize each token once per routed expert and scatter it to the physical +/// rows assigned by the routing: packed FP4 `[capacity * group, hidden / 2]` +/// and E4M3 scales `[capacity, hidden / 16 * 128]` in the 128x4 layout. +/// `ids` are the `[tokens * topk]` local experts and `global_scale` the +/// `[experts]` FP32 activation global scales. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, ids: zml.Tensor, global_scale: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(@rem(cfg.hidden, 64) == 0); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map, .ids = ids, .global_scale = global_scale }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 2) }, .u8), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ words_per_cta, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .ids = .{ .ptr = cute.DType.i32 }, + .global_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + }); + const one = b.constant(.i32, 1); + const words = @divExact(cfg.hidden, 8); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, @as(i32, @intCast(@divTrunc(words + words_per_cta - 1, words_per_cta)))), b.constant(.i32, @as(i32, @intCast(cfg.tokens))), one }, + .block = .{ b.constant(.i32, words_per_cta), one, one }, + .dynamic_smem = b.kernelSmemSize(quantize_kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(quantize_kernel_name, config, .{ + b.recastPointer(host.x, .i32, .gmem, 16), host.route_map, host.ids, host.global_scale, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); +} + +/// Each thread quantizes eight consecutive values into one packed word, so +/// two lanes cover one 16-value NVFP4 block. +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + const words = @divExact(cfg.hidden, 8); + const routes = cfg.tokens * cfg.topk; + const a = try b.declareArgs(.{ + // Two BF16 values per word. + .x = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, @divExact(cfg.hidden, 2) } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .ids = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .global_scale = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.experts} } }, + // Eight FP4 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, words } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // The routing and the previous layer write what this grid reads. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(words_per_cta).add(tid); + // Out-of-range lanes still take part in the pair reduction. + const in_range = word.lt(@as(i32, @intCast(words))); + const safe_word = b.select(in_range, word, b.constant(.i32, 0)); + var x: [8]V = undefined; + for (0..4) |i| { + const pair = a.x.get(.{ token, safe_word.mul(4).add(@as(i32, @intCast(i))) }); + x[2 * i] = pair.shl(16).bitCast(.f32); + x[2 * i + 1] = pair.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32); + } + var maximum = x[0].abs(); + for (x[1..]) |value| maximum = maximum.maximum(value.abs()); + maximum = maximum.maximum(b.shuffleXor(maximum, 1)); + + const block = safe_word.div(2); + var in_range_scope = b.openIf(in_range); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + const expert = a.ids.get(.{route}); + var scheduled = b.openIf(row.ge(0).bitAnd(expert.ge(0))); + const scale = blockScale(b, maximum, a.global_scale.get(.{expert})); + var quantized: [8]V = undefined; + for (&quantized, x) |*q, value| q.* = value.mul(scale.multiplier); + a.q.set(.{ row, safe_word }, b.packFp4x8(quantized)); + + var leader = b.openIf(tid.rem(2).eq(0)); + a.s.set(.{ row.div(cfg.group_size), swizzledScaleOffset(row.rem(cfg.group_size), block) }, scale.bits.to(.i8)); + leader.close(); + scheduled.close(); + } + in_range_scope.close(); + // The rows this CTA owns are complete: release the persistent GEMM. + b.launchDependents(); +} + +test "NVFP4 CuTe quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .experts = 192, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + .direct = tokens == 1, + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cvt.rn.satfinite.e2m1x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig new file mode 100644 index 00000000..30062a3a --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -0,0 +1,1923 @@ +//! Source-level SM100 MXFP4 grouped GEMM. +//! +//! This file is the Zig CuTe-DSL counterpart of the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with its fused MoE epilogues. +//! It emits a complete CuTe program: the nested `cuda.kernel` consumes the +//! objects constructed by the public host `func.func`, and the host function +//! creates the block-scaled MMA objects, four TMA tensor maps, persistent +//! scheduler parameters, and the PDL-enabled launch. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +const std = @import("std"); +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const common = @import("common.zig"); +const typed = common.typed; +const gridTyped = common.gridTyped; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const PersistentWork = common.PersistentWork; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; +const fmt = common.fmt; +const view = common.view; + +const accumulator_stages = 2; + +/// Shared-memory plan of the Python kernel for one tile width, copied from +/// its generated `SharedStorage` (`smem.struct_fields`). The A/B ring depth +/// is whatever fits next to the two C epilogue stages; the fused epilogues +/// only use the start of `sC` as scratch. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + /// Bytes reserved at `c`. Only the epilogue scratch is used; the wide + /// prefill tiles keep just that instead of two full C stages. + c_bytes: ?i64 = null, + + pub fn of(comptime n: i64) Tile { + return switch (n) { + 8 => .{ + .n = 8, + .stages = 12, + .smem_bytes = 230400, + .tmem_columns = 64, + .empty_barriers = 96, + .accumulator_full = 192, + .accumulator_empty = 208, + .tmem_dealloc = 224, + .tmem_holding = 232, + .c = 1024, + .a = 9216, + .b = 205824, + .sfa = 218112, + .sfb = 224256, + }, + 16 => .{ + .n = 16, + .stages = 11, + .smem_bytes = 232448, + .tmem_columns = 64, + .empty_barriers = 88, + .accumulator_full = 176, + .accumulator_empty = 192, + .tmem_dealloc = 208, + .tmem_holding = 216, + .c = 1024, + .a = 17408, + .b = 197632, + .sfa = 220160, + .sfb = 226304, + }, + 32 => .{ + .n = 32, + .stages = 9, + .smem_bytes = 228352, + .tmem_columns = 128, + .empty_barriers = 72, + .accumulator_full = 144, + .accumulator_empty = 160, + .tmem_dealloc = 176, + .tmem_holding = 184, + .c = 1024, + .a = 33792, + .b = 181248, + .sfa = 218112, + .sfb = 223232, + }, + // Prefill tiles. One expert weight tile feeds 64 or 128 routed + // rows instead of 32, which is what keeps large batches from + // re-reading the expert weights once per group. They hold only + // the epilogue scratch at `c`, so the A/B ring stays deep. + 64 => .{ + .n = 64, + .stages = 8, + .smem_bytes = 206848, + .tmem_columns = 256, + .empty_barriers = 64, + .accumulator_full = 128, + .accumulator_empty = 144, + .tmem_dealloc = 160, + .tmem_holding = 168, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .sfa = 133120, + .sfb = 137216, + .b = 141312, + }, + 128 => .{ + .n = 128, + .stages = 6, + .smem_bytes = 204800, + .tmem_columns = 512, + .empty_barriers = 48, + .accumulator_full = 96, + .accumulator_empty = 112, + .tmem_dealloc = 128, + .tmem_holding = 136, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .b = 100352, + .sfa = 198656, + .sfb = 201728, + }, + else => @compileError("unsupported MXFP4 tile N"), + }; + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + const a_stage_bytes = 128 * 128; + const sf_stage_bytes = 512; + + fn bStageBytes(self: Tile) i64 { + return self.n * 128; + } + + fn cStageElements(self: Tile) i64 { + return 128 * self.n; + } + + /// TMA bytes landing in one A/B stage: unpacked FP4 A is counted as + /// 128x128 nibbles, then the FP8 B tile and both 512-byte scale atoms. + fn transactionBytes(self: Tile) u32 { + return @intCast(8192 + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", self.c_bytes orelse accumulator_stages * self.cStageElements() * 4, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: clamped SwiGLU of the interleaved gate/up rows, BF16 + /// rounding and the routing weight as in the reference model, then MXFP8 + /// quantization straight into the down GEMM's input and 128x4 scale + /// layouts. Python `fused=1`. + swiglu_mxfp8, + /// Down projection: BF16 rows stored in route order `[routes, m]`, + /// skipping padding rows. Python `fused=2`. BF16 halves the traffic of + /// this store and of the top-k reduction that reads it back; the routed + /// partial sums are rounded exactly like the reference finalize. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (4608 for gate/up, 5120 for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension (5120 for gate/up, 2304 for down). + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp applied to both halves of the fused SwiGLU. Emitted as an IR + /// constant, so a threaded value costs exactly what a literal did. + swiglu_limit: ?f32 = 10.0, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + /// `schedule` is then just the `[routes]` expert ids; its size and + /// active-count sections and `route_inverse` are not read. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile, of + /// which direct routing fills only the first. A tensor map whose row mode + /// had extent one would lose that mode altogether — the TMA atom infers a + /// zero stride for it — and every group would then read the same row. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; the kernel reads the first and last sections in place. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// entries of padding rows are never read. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_mxfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_mxfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{"route_inverse"}), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + weight: zml.Tensor, + schedule: zml.Tensor, + input_quant: zml.Tensor, + weight_scale: zml.Tensor, + input_scale: zml.Tensor, +}; + +/// Up projection with the SwiGLU + MXFP8 epilogue: the routed FP8 rows +/// `[groups * n, m / 2]` and their 128x4 scales `[groups, m / 64 * 128]`, +/// ready for the down GEMM. `routing_weights` is `[routes]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_mxfp8); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ + .q = .init(.{ cfg.routedRows(), activations }, .f8e4m3fn), + .s = .init(.{ cfg.groups, @divExact(activations, 32) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing FP32 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + }, .{ .output = .init(.{ cfg.routes, cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +/// The two projections share this kernel but get distinct symbols, so a +/// profile shows them as separate rows instead of one merged entry. +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_mxfp8 => "mxfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "mxfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported MXFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{1}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{2}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 1, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 2, 1 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 65536, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4_unpack_u8 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = @intCast(tile.bStageBytes() * 8), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e8m0fnu, .copy_bits = 4096, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts, in the same element units as Python. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1 }); + } + }; +} + +/// Device memrefs of the route-indexed epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + // Group sizes follow the group experts inside `schedule`. + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .q = b.memrefType(.f8e4m3fn, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, 128) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + // The active group count is the last schedule entry, 4-byte aligned. + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .n = 128 }); + + // Device ABI. The host function below constructs every non-buffer value. + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + }))); + + // Prefetch every tensor-map descriptor before the specialized warps begin + // the persistent pipeline. This is the work PDL buys us: it runs while the + // producing grid is still writing the operands. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + { + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); + } + // Everything below reads the schedule and the quantized activations the + // preceding grid produces, so the whole block waits here. + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + // The epilogue has written every row this CTA owns. + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + // XLA owns packed storage as signless bytes. Recast only inside the host + // program when constructing typed tensor maps. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, + .weight_scale = .{ .ptr = cute.DType.i32 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the five mainloop operands, the epilogue's +/// extra inputs and outputs, then the five problem-shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const route_inverse = 5; + const routing_weights = 6; + const output = switch (epilogue) { + .swiglu_mxfp8 => 7, + .route_rows => 6, + }; + const scales = 8; + const problem_m = switch (epilogue) { + .swiglu_mxfp8 => 9, + .route_rows => 7, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors, as Python's `PipelineTmaUmma` +/// consumers see them. Per-stage views are slices of these. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeView( + b.recastSwizzledPointer(storage.a, .i8, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A shared pointer"), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeView( + b.recastSwizzledPointer(storage.b, .f8e4m3fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B shared pointer"), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + // Keep the shared operands as staged tensors and select the current stage + // with CuTe slicing. This is the representation used by the Python + // PipelineTmaUmma implementation; rebuilding a memref from a runtime + // pointer loses the staged descriptor semantics at ring wrap. + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); + // One scale stage is 512 bytes, so slicing a stage out of the ring keeps + // that alignment, not the array's. + const sf_stage_align = Tile.sf_stage_bytes; + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` preserves a singleton CTA mode. Keep it in the copy + // views: it is part of the executable TMA atom's expected algebra even + // for a one-CTA cluster. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// Issue the four transactions that fill one mainloop stage. The caller is +/// already restricted to the TMA warp; one elected lane owns the mbarrier +/// arrival. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + comptime reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k_tile: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const dynamic = cute.AlgebraToken.dynamic; + const div_128: cute.ConstrainedDynamic = .{ .divisible_by = 128 }; + const div_n: cute.ConstrainedDynamic = .{ .divisible_by = @intCast(n) }; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const sf_copy_spec = b.layoutSpec( + .{ .{ 512, 1 }, .{1} }, + .{ .{ b.basis(1, 2, .{0}), 0 }, .{0} }, + ); + // Follow the same coordinate algebra as the Python kernel. TMA + // partitioning produces a tensor over (tile, M, K, expert/group); first + // select the output tile and expert/group, then select the K stage. The + // remaining iterator carries the tensor-map coordinates needed by copy. + const a_tile_k = b.slice( + partitioned_a_target, + .{ all, tile_m, all, expert }, + b.coordTensorTypeWithTuple(.{ 0, div_128, dynamic }, b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const a_tile = b.slice( + a_tile_k, + .{ all, k_tile }, + b.coordTensorTypeWithTuple(.{ div_128, div_128, dynamic }, b.layoutSpec( + .{.{ .{ 128, 128 }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const a_source = b.rebuildCoordTensorIterator(3, a_tile, .{ div_128, div_128, dynamic }, .{ 128, 128, 1 }, a_copy_layout); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value rather than a literal: a statically zero leaf collapses out + // of the coordinate tuple, and the TMA lowering then shifts the remaining + // tensor-map coordinates, silently reading group zero for every group. + const b_tile_n = b.constant(.i32, 0); + const b_tile_k = b.slice( + partitioned_b_target, + .{ all, b_tile_n, all, group }, + b.coordTensorTypeWithTuple(.{ 0, div_n, dynamic }, b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const b_tile = b.slice( + b_tile_k, + .{ all, k_tile }, + b.coordTensorTypeWithTuple(.{ div_128, div_n, dynamic }, b.layoutSpec( + .{.{ .{ 128, n }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const b_source = b.rebuildCoordTensorIterator(3, b_tile, .{ div_128, div_n, dynamic }, .{ 128, n, 1 }, b_copy_layout); + const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, b.constant(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + { + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + } + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + // The stage offset is a whole tile width, which `cute.add_offset` needs to + // know: it derives the result alignment from what it knows of the offset. + return b.addOffset( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// Warp 5 runs independently over every persistent output tile. Keep the +/// producer index and phase as loop-carried values across tiles, exactly like +/// CUTLASS PipelineState. Only the per-tile count is reset to zero. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.constant(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + var stages = b.openFor( + 0, + k_tiles, + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + } + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + } + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + } + + // PipelineTmaUmma.producer_tail: drain every empty barrier before the TMA + // warp exits so no outstanding tcgen05 arrival outlives the CTA. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +/// SM100 MXFP4 mainloop objects of one pipeline stage, produced by +/// `StagedMxfp4Mainloop.atStage`. The dependent CuTe result types are +/// centralized here; callers only provide tensors and the allocated +/// tensor-memory pointer. +const Mxfp4Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + } +}; + +/// Pipeline-wide SM100 MXFP4 mainloop objects. CUTLASS derives the UMMA and +/// S2T descriptors from the complete staged tensors once, then slices those +/// descriptors at the current PipelineState index. Keeping the stage mode in +/// the descriptor is required when the circular buffer wraps back to stage 0. +const StagedMxfp4Mainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + /// MMA tile N: accumulator columns per stage. + n: i64, + + pub fn atStage(self: StagedMxfp4Mainloop, b: *B, index: V, accumulator_tmem: V) Mxfp4Mainloop { + const all = cute.AlgebraToken.all; + const a = b.slice( + self.a, + .{ all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), + ); + const rhs = b.slice( + self.b, + .{ all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), + ); + const sfa_stage = b.slice( + self.sfa_source, + .{ all, all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), + ); + const sfb_stage = b.slice( + self.sfb_source, + .{ all, all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), + ); + const source_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, .{ 1, 1, 1 } }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, .{ 0, 0, 0 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); + const sfa_source = b.makeView( + b.getIter(sfa_stage, b.smemDescType()), + source_grouped_layout, + source_grouped_ty, + ); + const sfb_source = b.makeView( + b.getIter(sfb_stage, b.smemDescType()), + source_grouped_layout, + source_grouped_ty, + ); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +/// Build the same pipeline-wide descriptors as Python's +/// `Sm100BlockScaledPersistentDenseGemmKernel`: `make_fragment_A/B` and the +/// S2T partitions see the complete staged tensors. `atStage` performs the +/// per-iteration slicing afterwards. +fn makeMxfp4StagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMxfp4Mainloop { + // Descriptor strides are in 16-byte units: a 128x128 unpacked-FP4 A + // stage is 16 KiB and an Nx128 FP8 B stage is N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, 1024 })), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns, matching num_accumulator_tmem_cols/num_sfa_tmem_cols in Python. + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }); + const sf_layout = b.staticLayout(sf_spec); + const tmem_f32_ptr = b.ptrTy(.f32, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const tmem_sf_ptr = b.ptrTy(.f8e8m0fnu, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), tmem_f32_ptr); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), tmem_f32_ptr); + const sfa_ptr = b.recastIter(sfa_col, tmem_sf_ptr); + const sfb_ptr = b.recastIter(sfb_col, tmem_sf_ptr); + const sf_memref = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(sf_spec)); + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 1 } }, 1, 4, stages }, + .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, 512 }, + ))); + const tmem_filtered_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 1 } }, 1, 4 }, + .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }, + ))); + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom_ty = b.s2tCopyAtomType(.f8e8m0fnu, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType( + copy_atom_ty, + b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 512, 32 }, 128 } }), + &.{ b.layoutSpec(512, 1), b.layoutSpec(1, 0), b.layoutSpec(4, 1) }, + ); + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, 4, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 16, 1, 4, 0 }, 0 }, 0, 0, 0, 512 }, + ))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec( + .{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0, 32 }, + )); + const target_partition_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, 1, 1, 1 }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, 0, 0, 0 }, + ))); + const target_grouped_spec = b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, .{ 1, 1, 1 } }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, .{ 0, 0, 0 } }, + ); + const target_grouped_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(target_grouped_spec)); + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + // Match Python's construction order: derive A/B UMMA fragments and S2T + // scale descriptors from the complete staged tensors once. The K loop + // slices these compiler-derived descriptors by PipelineState.index. + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeMxfp4StagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + // The MMA warp never reads the group, only the tile count. + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + // Python peeks the first AB-full token before it blocks on an available + // accumulator stage. Preserve that ordering so the two independent + // pipelines enter the K loop with the same outstanding operations. + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + k_tiles, + 1, + // Match PipelineTmaUmma's Python loop-carried ABI exactly. The + // software-pipeline pass recognizes the leading try-wait token and + // keeps the MMA atom immediately after it. + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + } + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + commit_lane.close(); + } + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + } + + var full_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + full_lane.close(); + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + } + + // Match PipelineUmmaAsync.producer_tail. The state points at the next + // accumulator slot; advance to the last slot used by this producer and + // wait until the epilogue has released it before the MMA warp exits. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + /// `Config.groupRows`. + group_rows: i64 = 0, + /// `Config.swiglu_limit`. + swiglu_limit: ?f32 = 10.0, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m, + initial_work.n, + initial_work.group, + .i32, + .i32, + .i32, + initial_work.linear, + initial_work.executed, + }); + { + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // A warp lane holds one routed row, so tiles wider than a warp are drained + // 32 rows at a time. The accumulator stage is released once its last chunk + // is in registers. + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // Route metadata does not depend on the accumulator. + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + // Registers now own the accumulator values, so all 128 epilogue + // threads release this TMEM stage before the stores. + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); + switch (epilogue) { + .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + } + + var release_warp = b.openIf(warp.eq(0)); + { + // Permit release and deallocation are warp collective as well. + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + } + b.barrierCtaSyncAligned(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + { + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); + } +} + +/// E8M0 exponent of the power-of-two MX scale for a block maximum, rounded +/// up so that `maximum / scale <= 448`, and the matching inverse scale. +fn mxScale(b: *B, maximum: V) struct { exponent: V, inverse: V } { + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + return .{ .exponent = exponent, .inverse = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the group. Columns index it with `shuffleIdx`. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row (up projection only). + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + // Only the up projection applies routing weights. + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.constant(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. Wide tiles + // run one 32-row chunk at a time, so lane `l` holds row `32 * chunk + l`. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(chunk_rows), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i64, @intCast(chunk)) * chunk_rows); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.constant(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 +/// projection. Gate and up lanes are paired by XOR 8. Each gate thread +/// computes one SwiGLU activation per routed row; 32 consecutive activations +/// span two warps, whose partial maxima meet in shared memory. +/// +/// At decode sizes most rows of a group are padding, whose outputs are never +/// read. `count` is uniform across the CTA, so whole 8-row chunks of padding +/// are skipped; per-row branches would serialize the shuffle chains of +/// independent rows. Do not duplicate this body under a full-group branch: +/// ptxas then loses track of convergence at the tile loop's warp-uniform +/// shuffles, and lanes still outside the gate-lane stores read garbage. +fn storeTileSwiglu(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime chunk: usize) void { + // Direct routing has exactly one live row per group. + if (operands.direct) + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, 1, false, chunk) + else + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +const swiglu_chunk = 8; + +/// The first `live` rows of `storeTileSwiglu`; with `skip_padding`, chunks +/// of rows past `count` are skipped. +fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + // First routed row of this epilogue pass. + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(16).lt(8); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + // The TMA C staging buffer is unused by this epilogue. + const scratch_layout = b.layoutSpec(.{ 4, n }, .{ n, 1 }); + const scratch = b.makeView(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the warp maximum of column `c`. + var lane_maximum = b.constant(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + // Row 0 of an active group is always live. + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + // `@TypeOf` does not evaluate its operand. + var guard: ?@TypeOf(b.openIfElse(routed.count, types)) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), types) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference model rounds the projection to BF16. + var gate = value.to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + } + var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + up_value = up_value.maximum(-limit).minimum(limit); + } + // up * SiLU(gate), with the hardware approximations Python's + // fastmath selects: exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); + // Up lanes hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(then_values); + // Padding chunk: zero activations, maxima unchanged. The zero is + // emitted in the else block, where it is used. + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(else_values); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + var owner = b.openIf(lane.lt(n)); + { + scratch.set(.{ warp, lane }, lane_maximum); + owner.close(); + } + b.barrierCtaSyncAligned(1, 128); + + // Lane `c` turns the two half-block maxima of column `c` into its scale. + const pair = warp.bitAnd(~@as(i32, 1)); + const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); + const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); + // The next tile reuses the scratch maxima. + b.barrierCtaSyncAligned(1, 128); + + const activation_row = tid.div(16).mul(8).add(tid.rem(8)); + const activation_index = tile_m.mul(64).add(activation_row); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + var inverses: [swiglu_chunk]V = undefined; + for (inverses[0 .. last - first], first..) |*inverse, column| inverse.* = b.shuffleIdx(scale.inverse, @as(i32, @intCast(column))); + var gate_lane = b.openIf(is_gate); + { + for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { + const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); + operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); + } + gate_lane.close(); + } + if (guard) |*g| g.close(); + } + + // Warps 0 and 2 each own one 32-activation block of this tile. + var scale_lane = b.openIf(warp.rem(2).eq(0).bitAnd(lane.add(chunk_first).lt(routed.count))); + { + const block = tile_m.mul(2).add(warp.div(2)); + const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); + operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); + scale_lane.close(); + } +} + +/// Python `down_epilogue`: store each live routed row at its token/top-k +/// route, so the final reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + operands.rows.?.set(.{ routed.route, output_column }, columns[0].to(.bf16)); + return; + } + var routes: [32]V = undefined; + for (routes[0..columns.len], 0..) |*route, column| route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + for (columns, routes[0..columns.len], 0..) |value, route, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + operands.rows.?.set(.{ route, output_column }, value.to(.bf16)); + if (live) |*l| l.close(); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + // Allocate the complete Python SharedStorage object as one compiler-visible + // memref. `cute.kernel_smem_size` only accounts for memref allocations; + // treating every field as an architecture pointer left the launch at zero + // dynamic shared bytes and made the first TMA instruction fault. + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIter( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .f8e4m3fn, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e8m0fnu, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e8m0fnu, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // Reproduce the Python CuTe path for the two matrix operands: + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition. All dependent result types come from Zig layout + // objects; this kernel does not embed textual MLIR. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, 128); + const a_local_spec = b.layoutSpec( + .{ 128, 128, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + // The group tile covers all of B's rows, so the N-tile mode has extent one + // and its stride is inferred as zero. Direct routing is no different: the + // tensor map still spans a whole tile of rows, zero-filling all but one. + const b_local_spec = b.layoutSpec( + .{ n, 128, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 32 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 32 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartition( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartition( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Match CUTLASS PipelineTmaUmma/PipelineUmmaAsync initialization. Warp 0 + // owns initialization, and one elected lane initializes each complete + // barrier array. mbarrier.init is not warp collective: issuing it from + // every lane races on the same shared state and breaks the first phase + // transition when the A/B ring wraps. + var init_warp = b.openIf(warp.eq(0)); + { + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + { + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } + } + b.fenceMbarrierInit(); + init_warp.close(); + } + b.syncThreads(); + + // Each role walks the same persistent tile sequence independently. The + // AB and accumulator mbarriers carry all cross-warp dependencies. + var tma_warp = b.openIf(warp.eq(5)); + { + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), + else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), + } + tma_warp.close(); + } + + var mma_warp = b.openIf(warp.eq(4)); + { + // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. + b.barrierCtaSyncAligned(2, 160); + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), + else => unreachable, + } + mma_warp.close(); + } + + var epilogue_warps = b.openIf(warp.lt(4)); + { + // TmemAllocator.allocate is issued collectively by warp 0. The four + // epilogue warps then meet the MMA warp at named barrier 2. + var alloc_warp = b.openIf(warp.eq(0)); + { + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + } + b.barrierCtaSyncAligned(2, 160); + switch (epilogue) { + .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = view(b, d.routing_weights), + .q = view(b, d.q), + .s = view(b, d.s), + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }), + .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .rows = view(b, d.rows), + .direct = cfg.direct, + }), + } + epilogue_warps.close(); + } +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = Tile.sf_stage_bytes }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + // `schedule` is `[group experts | group sizes | active count]`. + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffset( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.arg(host_arg.input_quant), b.staticLayout(b_layout)); + // Match Python's host construction instead of materializing these two + // layouts with `cute.static`. In particular, SFB's M extent is one; CuTe + // must still retain that mode while encoding the tensor map. + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 32, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // The Zig dialect binding exposes the zero-based form of Python CuTe's + // `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + // Weight scales step M atoms first; activation scales retain K-major order. + const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); + const sfa_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, + ); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, 32 }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, b.basis(1, 1, .{ 1, 0, 0 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.constant(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, + }; + const activations = @divExact(cfg.m, 2); + const group_sizes = b.makeTensorView( + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const launch = switch (epilogue) { + .swiglu_mxfp8 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "MXFP4 GEMM programs emit their host launch for every tile width" { + inline for (.{ 8, 16, 32 }) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_mxfp8, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "make_non_exec_tiled_tma_load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x32", .{n})) != null); + } + } +} diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig new file mode 100644 index 00000000..0de45007 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -0,0 +1,1679 @@ +//! Source-level SM100 NVFP4 grouped GEMM. +//! +//! The NVFP4 counterpart of `persistent_mxfp4.zig`, following the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with FP4 x FP4 operands, E4M3 +//! scale factors every 16 elements and the `kind::mxf4nvf4.block16` MMA. Both +//! operands are packed FP4 in global and shared memory (TMA format `U4`); one +//! pipeline stage holds a 256-deep K slice, consumed by four K = 64 MMAs. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +//! The pipeline, scheduler and warp roles are shared with the MXFP4 kernel. +const std = @import("std"); +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const common = @import("common.zig"); +const nvfp4 = @import("nvfp4.zig"); + +const accumulator_stages = 2; +/// K elements of one pipeline stage: four K = 64 FP4 MMAs. +pub const k_tile = 256; +/// Largest dynamic shared-memory allocation of an SM100 CTA. +const smem_capacity = 232448; +const mma_kind: cute.BlockScaledMmaConfig.Kind = .nvfp4_nvfp4; + +/// Shared-memory plan for one tile width, laid out like the Python +/// `SharedStorage`: barriers, epilogue scratch, then the A, B, SFA and SFB +/// rings. The ring is as deep as the capacity allows. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + + /// 128x256 packed FP4. + const a_stage_bytes = 128 * k_tile / 2; + /// One 128x16 E4M3 scale tile: four 512-byte 128x4 atoms. + const sf_stage_bytes = 128 * k_tile / 16; + const c_bytes = 512; + const header_bytes = 1024; + + pub fn of(comptime n: i64) Tile { + comptime { + std.debug.assert(n >= 8 and n <= 128 and @rem(n, 8) == 0); + const per_stage = a_stage_bytes + @divExact(n * k_tile, 2) + 2 * sf_stage_bytes; + const stages: usize = @intCast(@divTrunc(smem_capacity - header_bytes - c_bytes, per_stage)); + const s: i64 = @intCast(stages); + const a = header_bytes + 1024; + const b = a + s * a_stage_bytes; + const sfa = b + @divExact(s * n * k_tile, 2); + const sfb = sfa + s * sf_stage_bytes; + // Accumulators, then 16 SFA and 16 SFB columns. + const columns = std.math.ceilPowerOfTwo(u16, @max(32, accumulator_stages * n + 32)) catch unreachable; + return .{ + .n = n, + .stages = stages, + .smem_bytes = sfb + s * sf_stage_bytes, + .tmem_columns = columns, + .empty_barriers = 8 * s, + .accumulator_full = 16 * s, + .accumulator_empty = 16 * s + 16, + .tmem_dealloc = 16 * s + 32, + .tmem_holding = 16 * s + 40, + .c = header_bytes, + .a = a, + .b = b, + .sfa = sfa, + .sfb = sfb, + }; + } + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + fn bStageBytes(self: Tile) i64 { + return @divExact(self.n * k_tile, 2); + } + + /// TMA bytes landing in one stage: packed A and B, then both scale tiles. + fn transactionBytes(self: Tile) u32 { + return @intCast(a_stage_bytes + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", c_bytes, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: per-expert dequantization scale, BF16 rounding, + /// SwiGLU of the interleaved gate/up rows (optionally clamped), the + /// routing weight when it precedes the down projection, BF16 rounding, + /// then NVFP4 quantization (E4M3 scale per 16 activations, scaled by the + /// per-expert activation global scale) straight into the down GEMM's + /// packed input and 128x4 scale layouts. + swiglu_nvfp4, + /// Down projection: per-expert dequantization scale and, when it follows + /// the down projection, the routing weight, then BF16 rows stored in + /// route order `[routes, m]`, skipping padding rows. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (2 x intermediate for gate/up, hidden for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension, a multiple of `k_tile`. + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp of both halves of the fused SwiGLU, if any. + swiglu_limit: ?f32 = null, + /// Multiply rows by their routing weight in this projection's epilogue. + routing_weights: bool = false, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; `alpha` the per-expert FP32 dequantization scale of the +/// GEMM output. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", + "alpha", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// `act_scale` is the per-expert global scale of the down GEMM's NVFP4 input. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_nvfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights", "act_scale" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_nvfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + /// `[experts, m, k / 2]` packed FP4. + weight: zml.Tensor, + schedule: zml.Tensor, + /// `[groups * n, k / 2]` packed FP4. + input_quant: zml.Tensor, + /// `[experts, m, k / 16]` E4M3 bytes in the 128x4 layout. + weight_scale: zml.Tensor, + /// `[groups, k / 16 * 128]` E4M3 bytes in the 128x4 layout. + input_scale: zml.Tensor, + /// `[experts]` FP32. + alpha: zml.Tensor, +}; + +/// Up projection with the SwiGLU + NVFP4 epilogue: the routed packed FP4 +/// rows `[groups * n, m / 4]` and their 128x4 E4M3 scales +/// `[groups, m / 32 * 128]`, ready for the down GEMM. `routing_weights` is +/// `[routes]` FP32 and `act_scale` `[experts]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor, act_scale: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_nvfp4); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + .act_scale = act_scale, + }, .{ + .q = .init(.{ cfg.routedRows(), @divExact(activations, 2) }, .u8), + .s = .init(.{ cfg.groups, @divExact(activations, 16) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing BF16 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ .output = .init(.{ @max(cfg.routes, 1), cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +const typed = common.typed; +const gridTyped = common.gridTyped; +const fmt = common.fmt; +const view = common.view; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; + +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_nvfp4 => "nvfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "nvfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +pub const tile_widths = [_]i64{ 8, 16, 32, 64, 128 }; + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported NVFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch, as the +/// CuTe DSL derives them for FP4 x FP4 with 16-element E4M3 scale vectors. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + /// TMA coordinates of the 128x4 scale layout: byte in the atom, K + /// atom, M block, L. + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{2}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{1}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 4, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 2, 1 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 128 * k_tile * 4, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = @intCast(n * k_tile * 4), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = Tile.sf_stage_bytes * 8, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts in FP4 elements for A/B, bytes for scales. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512 }); + } + fn fp4SmemPtrTy(b: *B) *const mlir.Type { + return b.swizzledPtrTy(.f4e2m1fn, .smem, 1024, ab_swizzle) catch @panic("bad FP4 shared pointer"); + } + }; +} + +/// Device memrefs of the epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + per_expert: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .per_expert = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.experts, 1))), + // Eight packed FP4 activations per word. + .q = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, k_tile) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = 128 }); + + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .act_scale = typed(et.per_expert), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + }))); + + // Prefetch every tensor-map descriptor while the producing grid drains. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i8 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + .alpha = .{ .ptr = cute.DType.f32 }, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .act_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the six mainloop operands, the route operands, +/// the epilogue's extra inputs and outputs, then the five shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const alpha = 5; + const route_inverse = 6; + const routing_weights = 7; + const act_scale = 8; + const output = switch (epilogue) { + .swiglu_nvfp4 => 9, + .route_rows => 8, + }; + const scales = 10; + const problem_m = switch (epilogue) { + .swiglu_nvfp4 => 11, + .route_rows => 9, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors. Per-stage views are slices. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeView( + b.recastSwizzledPointer(storage.a, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeView( + b.recastSwizzledPointer(storage.b, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, false)))); + const sf_stage_align = 1024; + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` keeps a singleton CTA mode; the copy views keep it too. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// The TMA coordinates of one stage: slice the TMA-partitioned operand at +/// its output tile and L index, then at the K stage, and rebuild the +/// iterator as the Python front end does. +fn tmaSource(b: *B, partitioned: cute.View, tile_coord: anytype, k: V, comptime rows: i64, reduction_tiles: usize, copy_layout: cute.View) cute.View { + const all = cute.AlgebraToken.all; + const tile_k = b.slice( + partitioned, + tile_coord, + b.coordTensorTypeWithTuple(.{ 0, cute.ConstrainedDynamic{ .divisible_by = rows }, cute.AlgebraToken.dynamic }, b.layoutSpec( + .{ .{ .{ k_tile, rows }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(k_tile, 1, .{0}) }, + )), + ); + const div_k = cute.ConstrainedDynamic{ .divisible_by = k_tile }; + const div_rows = cute.ConstrainedDynamic{ .divisible_by = rows }; + const dynamic = cute.AlgebraToken.dynamic; + const stage = b.slice( + tile_k, + .{ all, k }, + b.coordTensorTypeWithTuple(.{ div_k, div_rows, dynamic }, b.layoutSpec( + .{.{ .{ k_tile, rows }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + return b.rebuildCoordTensorIterator(3, stage, .{ div_k, div_rows, dynamic }, .{ k_tile, rows, 1 }, copy_layout); +} + +/// Issue the four transactions that fill one mainloop stage. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + // One stage moves four 512-byte atoms, one per K = 64 slice. + const sf_copy_spec = b.layoutSpec( + .{ .{ .{ 512, 4 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 2, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + ); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value: a static zero leaf collapses out of the coordinate tuple + // and shifts the remaining tensor-map coordinates. + const b_tile_n = b.constant(.i32, 0); + const a_source = tmaSource(b, partitioned_a_target, .{ all, tile_m, all, expert }, k, 128, reduction_tiles, a_copy_layout); + const b_source = tmaSource(b, partitioned_b_target, .{ all, b_tile_n, all, group }, k, n, reduction_tiles, b_copy_layout); + // Scale coordinates: (byte, K atom, M block, L); a stage covers 4 K atoms. + const k_atom = k.mul(4); + const sfa_source = b.makeCoordTensor(.{ 0, k_atom, tile_m, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, k_atom, b.constant(.i32, 0), group }, b.staticLayout(sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + return b.addOffset( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// Warp 5 runs independently over every persistent output tile. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.constant(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + + // PipelineTmaUmma.producer_tail: drain every empty barrier before exit. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +/// NVFP4 mainloop objects of one pipeline stage. +const Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + } +}; + +/// Pipeline-wide UMMA and S2T descriptors derived from the complete staged +/// tensors once; `atStage` slices them at the current stage index. +const StagedMainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + n: i64, + + pub fn atStage(self: StagedMainloop, b: *B, index: V, accumulator_tmem: V) Mainloop { + const all = cute.AlgebraToken.all; + const a = b.slice(self.a, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const rhs = b.slice(self.b, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const sfa_stage = b.slice(self.sfa_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const sfb_stage = b.slice(self.sfb_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const source_grouped_spec = b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, .{ 0, 0, 32 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); + const sfa_source = b.makeView(b.getIter(sfa_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const sfb_source = b.makeView(b.getIter(sfb_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +fn makeStagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMainloop { + // Descriptor strides are in 16-byte units: one 64-deep K slice of packed + // FP4 is 32 bytes, a 128x256 A stage 16 KiB and an Nx256 B stage N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, Tile.a_stage_bytes / 16 })), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns: four K = 64 slices of four E4M3 scales per lane each. + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }); + const sf_layout = b.staticLayout(sf_spec); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfa_ptr = b.recastIter(sfa_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sfb_ptr = b.recastIter(sfb_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sf_memref = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(sf_spec)); + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, 2048 }))); + const tmem_filtered_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }))); + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom_ty = b.s2tCopyAtomType(.f8e4m3fn, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType(copy_atom_ty, b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 32, 128 }, 512 } }), &.{ b.layoutSpec(.{ 32, 4, 16 }, .{ 1, 512, 32 }), b.layoutSpec(1, 0), b.layoutSpec(1, 0) }); + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4, 4 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 16, 4, 4 }, 0 }, 0 }, 0, 0, 512, 2048 }))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32, 128 })); + const target_partition_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, 0, 0, 16 }))); + const target_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, .{ 0, 0, 16 } }); + const target_grouped_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(target_grouped_spec)); + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeStagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + commit_lane.close(); + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + + var full_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + full_lane.close(); + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + + // PipelineUmmaAsync.producer_tail. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + expert_ids: cute.View, + alpha: cute.View, + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + act_scale: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + group_rows: i64 = 0, + swiglu_limit: ?f32 = null, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m.type_(), + initial_work.n.type_(), + initial_work.group.type_(), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + initial_work.linear.type_(), + initial_work.executed.type_(), + }); + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + // Per-expert scalars do not depend on the accumulator. + const expert = operands.expert_ids.get(.{group}, .i32); + const alpha = operands.alpha.get(.{expert}, .f32); + const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); + switch (epilogue) { + .swiglu_nvfp4 => storeTileSwiglu(b, tile, operands, routed, tile_m, group, act_scale.?, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + + var release_warp = b.openIf(warp.eq(0)); + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + b.barrierCtaSyncAligned(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the current chunk. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row, when this projection applies it. + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.constant(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.constant(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +const swiglu_chunk = 8; + +/// Thread `t` owns output row `128 * tile_m + t` of the interleaved +/// projection: even threads hold a gate and their odd neighbour the matching +/// up value, so each even thread computes one SwiGLU activation per routed +/// row. A warp's 16 activations are exactly one NVFP4 block, whose maximum is +/// a single warp reduction. Activation `a` of row `r` lands in nibble `a` of +/// row `r` of `q`, and its scale at the 128x4 position of `(r, a / 16)`. +fn storeTileSwiglu(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime chunk: usize) void { + if (operands.direct) + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, 1, false, chunk) + else + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +fn swigluRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(2).eq(0); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the block maximum of column `c`. + var lane_maximum = b.constant(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference rounds the projection to BF16. + var gate = value.to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + up_value = up_value.maximum(-limit).minimum(limit); + } + // up * SiLU(gate), exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + if (routed.weight) |weight| result = result.mul(b.shuffleIdx(weight, c)); + // The NVFP4 quantizer reads BF16 activations. + result = result.to(.bf16).to(.f32); + // Odd threads hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(tupleOf(then_values)); + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(tupleOf(else_values)); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + const scale = nvfp4.blockScale(b, lane_maximum, act_scale); + + // Lane `16 h` writes the word of activations `16 * warp + 8 h ..`. + const word = tile_m.mul(8).add(warp.mul(2)).add(lane.div(16)); + const word_lane = lane.rem(16).eq(0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + for (activations[first..last], first..) |activation, column| { + const c: i32 = @intCast(column); + const quantized = activation.mul(b.shuffleIdx(scale.multiplier, c)); + // Gate lane `4i` pairs with gate lane `4i + 2`, then pairs of + // bytes and pairs of halves meet at lanes `8i` and `16i`. + const pair = b.packFp4x2(quantized, b.shuffleXor(quantized, 2)); + const half = pair.bitOr(b.shuffleXor(pair, 4).shl(8)); + const packed_word = half.bitOr(b.shuffleXor(half, 8).shl(16)); + var writer = b.openIf(word_lane); + operands.q.?.set(.{ first_row.add(chunk_first + c), word }, packed_word); + writer.close(); + } + if (guard) |*g| g.close(); + } + + // Lane `c` stores the scale of row `c`, block `4 * tile_m + warp`. + var scale_lane = b.openIf(lane.lt(@as(i32, @intCast(n))).bitAnd(lane.add(chunk_first).lt(routed.count))); + const swizzled = tile_m.mul(512).add(lane.mul(16)).add(chunk * 4).add(warp); + operands.s.?.set(.{ group, swizzled }, scale.bits.to(.i8)); + scale_lane.close(); +} + +/// Store each live routed row at its token/top-k route, so the final +/// reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + const value = if (routed.weight) |weight| columns[0].mul(weight) else columns[0]; + operands.rows.?.set(.{ routed.route, output_column }, value.to(.bf16)); + return; + } + var routes: [32]V = undefined; + var weights: [32]?V = @splat(null); + for (routes[0..columns.len], weights[0..columns.len], 0..) |*route, *weight, column| { + route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + if (routed.weight) |w| weight.* = b.shuffleIdx(w, @as(i32, @intCast(column))); + } + for (columns, routes[0..columns.len], weights[0..columns.len], 0..) |value, route, weight, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + const scaled = if (weight) |w| value.mul(w) else value; + operands.rows.?.set(.{ route, output_column }, scaled.to(.bf16)); + if (live) |*l| l.close(); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIter( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .i8, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e4m3fn, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e4m3fn, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition, with the result types of the CuTe DSL. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, k_tile); + const m_step = b.basis(128, 1, .{1}); + const k_step = b.basis(k_tile, 1, .{0}); + const a_local_spec = b.layoutSpec( + .{ 128, k_tile, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_local_spec = b.layoutSpec( + .{ n, k_tile, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, k_tile }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, k_tile }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 64 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 64 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, m_step, k_step, b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, 2 * Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, 2 * tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartition( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartition( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Warp 0 initializes the barriers; one elected lane per barrier array. + var init_warp = b.openIf(warp.eq(0)); + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } + b.fenceMbarrierInit(); + init_warp.close(); + b.syncThreads(); + + const reduction_tiles: usize = @intCast(k_tiles); + var tma_warp = b.openIf(warp.eq(5)); + runTmaProducer(b, tile, reduction_tiles, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb); + tma_warp.close(); + + var mma_warp = b.openIf(warp.eq(4)); + b.barrierCtaSyncAligned(2, 160); + runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); + mma_warp.close(); + + var epilogue_warps = b.openIf(warp.lt(4)); + var alloc_warp = b.openIf(warp.eq(0)); + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + b.barrierCtaSyncAligned(2, 160); + const shared_operands: EpilogueOperands = .{ + .expert_ids = expert_ids, + .alpha = view(b, d.alpha), + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = if (cfg.routing_weights) view(b, d.routing_weights) else null, + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }; + var operands = shared_operands; + switch (epilogue) { + .swiglu_nvfp4 => { + operands.act_scale = view(b, d.act_scale); + operands.q = view(b, d.q); + operands.s = view(b, d.s); + }, + .route_rows => operands.rows = view(b, d.rows), + } + runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, operands); + epilogue_warps.close(); +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = 512 }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + // XLA owns packed storage as signless bytes; recast to FP4 here. + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffset( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_quant), .f4e2m1fn, .gmem, 16), b.staticLayout(b_layout)); + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 16, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // Zero-based form of Python CuTe's `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e4m3fn, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e4m3fn, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, .{ 16, 4 } }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, .{ b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 1, 0, 1 }) } }, 0, b.basis(1, 1, .{ 1, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.constant(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const group_sizes = b.makeTensorView( + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const per_expert = b.staticLayout(b.layoutSpec(cfg.experts, 1)); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, b.makeTensorView(b.arg(host_arg.alpha), per_expert), group_sizes, b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + }; + const activations = @divExact(cfg.m, 2); + const launch = switch (epilogue) { + .swiglu_nvfp4 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.act_scale), per_expert), + b.makeTensorView(b.recastPointer(b.arg(host_arg.output), .i32, .gmem, 16), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "NVFP4 GEMM programs emit their host launch for every tile width" { + inline for (tile_widths) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_nvfp4, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + .routing_weights = program[1] == .route_rows, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tma_format = U4") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x64", .{n})) != null); + } + } +} diff --git a/zml/kernels/moe/cute/sm10x/reduce.zig b/zml/kernels/moe/cute/sm10x/reduce.zig new file mode 100644 index 00000000..63244ac4 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/reduce.zig @@ -0,0 +1,124 @@ +//! CuTe top-k reduction behind the persistent MoE GEMMs: sums each token's +//! BF16 route rows. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const Config = struct { + tokens: i64, + hidden: i64, + topk: i64, + columns_per_cta: i64, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute (see `mxfp4.zig`). +const Topk = cute.Program(Config, .{ + .name = "moe_cute_reduce_topk", + .inputs = &.{ "d", "routes" }, + .outputs = &.{"y"}, + .run = topkProgram, +}); + +const topk_kernel_name = "moe_cute_reduce_topk_kernel"; + +/// Amortize launch overhead for large prefills while retaining enough CTAs +/// to saturate the GPU for decode and small batches. +pub fn columnsPerCta(tokens: i64, hidden: i64) i64 { + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + // Two columns per thread is the smallest tile: 128 threads, one 32-bit + // word each. + return 256; +} + +/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. +/// Routing weights were already applied before the down projection. +/// `routes` is negative for routes this rank does not compute (expert +/// parallelism): their rows were never written. +pub fn topk(d: zml.Tensor, routes: zml.Tensor, cfg: Config) zml.Tensor { + return Topk.call( + .{ .d = d, .routes = routes }, + .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, + .{ .cfg = cfg }, + ).y; +} + +/// Each thread owns two adjacent BF16 columns, read and written as one +/// 32-bit word: a warp then moves a full 128-byte line per access. +fn topkProgram(b: *B, cfg: Config) cute.FinishError!void { + b.beginFunction(topk_kernel_name, .cuda_kernel); + try topkKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(Topk.name, .host); + // The kernel reads and writes two BF16 columns per 32-bit word. + const host = try b.declareArgs(.{ + .d = .{ .ptr = cute.DType.bf16 }, + .routes = .{ .ptr = cute.DType.i32 }, + .y = .{ .ptr = cute.DType.bf16 }, + }); + try launchBoundary(b, topk_kernel_name, .{ + @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), + @intCast(cfg.tokens), + 1, + }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); + b.endFunction(null); +} + +fn topkKernel(b: *B, cfg: Config) cute.FinishError!void { + const pairs = @divExact(cfg.hidden, 2); + const a = try b.declareArgs(.{ + .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, + .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, + }); + + // The down projection's rows arrive from a grid this one may overlap. + b.waitForDependency(); + const token = b.blockIdx().y; + const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); + const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); + for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { + const pair = first_pair.add(chunk * 128); + var in_bounds = b.openIf(pair.lt(pairs)); + var low = b.constant(.f32, 0); + var high = b.constant(.f32, 0); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + // Routes of other expert-parallel ranks were never written. + var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); + const word = a.d.get(.{ route, pair }); + computed.yieldThen(.{ + low.add(word.shl(16).bitCast(.f32)), + high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), + }); + computed.yieldElse(.{ low, high }); + low = computed.results[0]; + high = computed.results[1]; + } + const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); + const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); + a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); + in_bounds.close(); + } + b.launchDependents(); +} + +test "CuTe top-k reduction emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try Topk.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .columns_per_cta = columnsPerCta(tokens, 5120), + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); + } +} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig new file mode 100644 index 00000000..92317c7c --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -0,0 +1,355 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const Attributes = contract.Attributes; +const runtime = @import("runtime.zig"); + +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32, + activation: Activation, + enable_pdl: bool, + gemm1_tactic: i32, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32, + + pub fn fromGlobalOptions(opts: GlobalOptions) Options { + return .{ + .workspace_query_device = 0, + .activation = opts.activation, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, + }; + } +}; + +pub fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + +const Inputs = contract.Inputs(zml.Tensor); + +const Outputs = contract.Outputs(zml.Shape); + +fn currentRunners() !*runtime.Runners { + const platform = zml.Compiler.current().platform; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: Activation) i32 { + return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + }; +} + +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; + + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, + ); +} + +fn computeCapability(platform: *const zml.Platform) !u16 { + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: Activation, +) zml.Shape { + const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 2 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 2 or + topk_ids.rank() != 2) + { + return error.InvalidShape; + } + + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + }; + const top_k = topk_ids.dim(.topk); + + if (tokens <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or + !fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = tokens, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, + options, + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + + attributes.runners = @intFromPtr(runners); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const context = runtime.makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + runtime.checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); + + const result = call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const topk_ids = routing.localIds(); + // The runner ignores routes whose local expert id is -1. + const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); + + const runner_options: Options = .fromGlobalOptions(opts); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); + + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); +} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/fly/BUILD.bazel b/zml/kernels/moe/fly/BUILD.bazel new file mode 100644 index 00000000..cb3b2ff4 --- /dev/null +++ b/zml/kernels/moe/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "moe.zig", + ], +) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig new file mode 100644 index 00000000..ba3d3c00 --- /dev/null +++ b/zml/kernels/moe/fly/fly.zig @@ -0,0 +1,294 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const fly = @import("moe.zig"); + +const log = std.log.scoped(.@"zml/moe/fused_experts"); + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + hidden_states: Tensor, + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "fly backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "fly backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "fly backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "fly backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "fly backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "fly backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "fly backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (expert_map_) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (expert_map_) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = gate_up_.bias, + .input_scale = input_scale, + .weight_scale = gate_up_.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = opts.activation.applyActivation(gate_up_out, layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = down_.bias, + .input_scale = down_input_scale, + .weight_scale = down_.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +// TODO(Corentin): unused for now +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { + if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or + down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or + opts.routing_weight_placement != .before_down) return false; + const gate_up = gate_up_.weight; + const down = down_.weight; + if (gate_up.rank() != 3 or down.rank() != 3) return false; + const gate_up_scale = gate_up_.quantizationScales() orelse return false; + const down_scale = down_.quantizationScales() orelse return false; + const experts = routing.global_num_experts; + if (experts < 1 or experts > 512) return false; + var max_width: i64 = hidden_states.dim(.d); + inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { + const weight, const scale = projection; + const packing_factor: i64 = switch (weight.dtype()) { + .u8, .i8 => 2, + .f4e2m1 => 1, + else => return false, + }; + const n = weight.dim(1); + const k = weight.dim(2) * packing_factor; + if (!fly.Mfma.supportsShape(n, k)) return false; + max_width = @max(max_width, n); + if (weight.dim(0) < 1 or weight.dim(0) > 512 or + scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != @divExact(k, 32)) return false; + switch (scale.dtype()) { + .u8, .i8, .f8e8m0 => {}, + else => return false, + } + } + const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); + return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { + std.debug.assert(scheme == .mxfp4); + return .{ x.convert(.bf16), null }; +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) zml.Tensor { + zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); + const weight = if (opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + return fly.Mfma.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + std.debug.assert(num_experts > 0 and num_experts <= 512); + std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + // GEMV handles up to six routes; MFMA shares weights across sorted rows. + const naive_block_assignment = num_assignments <= 6; + const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else blk: { + const flat_experts = ids.reshape(.{ .g = num_assignments }); + const aligned = fly.Align.call( + flat_experts, + Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), + block_size_m, + ); + const sorted = fly.Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); + break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; + }; + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig new file mode 100644 index 00000000..3cb82b96 --- /dev/null +++ b/zml/kernels/moe/fly/moe.zig @@ -0,0 +1,747 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const fly = zml.kernel.fly; +const Builder = fly.Builder; +const Value = fly.Value; + +pub const Align = struct { + const Result = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_pad: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { + for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { + if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; + } + const num_experts = cumsum.dim(0) - 1; + if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or + block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; + const capacity_bound = if (ids.dim(0) < num_experts) + ids.dim(0) * block_size + else + @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; + return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and + sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and + experts.dim(0) == @divTrunc(sorted.dim(0), block_size); + } + + /// All four outputs alias their corresponding input buffers. Invalid expert IDs + /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS + /// let each thread find its output block's expert without scanning every expert. + pub fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { + std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); + const cfg: Config = .{ + .routes = ids.dim(0), + .experts = cumsum.dim(0) - 1, + .capacity = sorted.dim(0), + .blocks = experts.dim(0), + .block_size = block_size, + .padding = block_size - 1, + }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 512, + .grid = .{ 2, 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + capacity: i64, + blocks: i64, + block_size: i64, + padding: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_moe_align", + .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const tid = b.threadId(.x); + const tid64 = tid.to(.i64); + const zero = b.constant(.i32, 0); + const one = b.constant(.i32, 1); + const experts = b.constant(.i32, cfg.experts); + // Distinct arrays match the histogram, inclusive ends and wave totals. + const counts = b.sharedArray(.i32, 512, 16); + const ends = b.sharedArray(.i32, 512, 16); + const waves = b.sharedArray(.i32, 8, 16); + var fill = b.openIfElse(b.blockId(.x).ne(0), .{}); + { + var positions = b.openFor(tid64, cfg.capacity, 512, .{}); + { + args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); + positions.close(); + } + fill.elseBranch(); + } + { + counts.set(tid, zero); + b.barrier(); + var routes = b.openFor(tid64, cfg.routes, 512, .{}); + { + const id = args.ids.at(routes.iv); + var valid_id = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); + valid_id.close(); + } + routes.close(); + } + b.barrier(); + + const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + var prefix = padded; + inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { + const peer = prefix.shuffle(.up, shift, 64); + prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + } + var last_lane = b.openIf(lane.eq(63)); + { + waves.set(wave, prefix); + last_lane.close(); + } + b.barrier(); + + var sums = b.openFor(zero, 8, 1, .{ zero, zero }); + { + const value = waves.at(sums.iv); + const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); + const total = sums.carried[1].add(value); + sums.yield(.{ preceding, total }); + } + const end = sums.results[0].add(prefix); + const start = end.sub(padded); + ends.set(tid, end); + var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); + { + args.cumsum.set(tid64, start); + has_expert.close(); + } + var first = b.openIf(tid.eq(0)); + { + args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); + args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); + first.close(); + } + b.barrier(); + + var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); + { + const offset = blocks.iv.to(.i32).mul(cfg.block_size); + var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero}); + { + // Upper bound of inclusive expert ends handles empty experts without a scan. + var bounds = b.openWhile(.{ zero, experts }, .{ zero, zero }); + { + bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); + } + { + const lo = bounds.after_carried[0]; + const hi = bounds.after_carried[1]; + const mid = lo.add(hi).shrU(1); + const advance = ends.at(mid).cmpUnsigned(.le, offset); + bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); + } + block_expert.yieldThen(.{bounds.results[0]}); + } + { + block_expert.yieldElse(.{b.constant(.i32, -1)}); + } + args.expert_ids.set(blocks.iv, block_expert.results[0]); + blocks.close(); + } + fill.close(); + } + } +}; + +pub const Sort = struct { + const Result = struct { + sorted_token_ids: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { + return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and + ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and + cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and + ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); + } + + /// cumsum contains one starting offset per expert followed by the total padded + /// capacity. Each valid route atomically advances its expert's cursor; ordering + /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. + pub fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { + std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); + if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; + const routes = ids.dim(0); + const grid = @min(@divTrunc(routes + 255, 256), 65535); + const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(grid), 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + step: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_count_and_sort_expert_tokens", + .inputs = &.{ "ids", "sorted", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); + var routes = b.openFor(start, cfg.routes, cfg.step, .{}); + { + const id = args.ids.at(routes.iv); + var valid = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); + const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); + args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); + valid.close(); + } + routes.close(); + } + } +}; + +pub const Mfma = struct { + /// MFMA loads full 64-column by 128-reduction tiles without tail masks. + pub fn supportsShape(n: i64, k: i64) bool { + return n > 0 and k > 0 and @mod(n, 64) == 0 and @mod(k, 128) == 0; + } + + /// The caller owns route localization, activation and reduction. + /// `expert_ids` contains local bank indices, including -1 for remote experts. + /// Sorted routes must contain a real prefix followed by padding per expert. + /// `block_m` is their alignment; native tiles contain 16 or 32 rows. + pub fn call( + input: Tensor, + weight: Tensor, + scales: Tensor, + sorted_ids: Tensor, + expert_ids: Tensor, + padded_count: Tensor, + output: zml.Shape, + topk: usize, + naive: bool, + block_m: usize, + ) Tensor { + std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and + (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and + sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and + padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and + topk > 0 and block_m > 0 and block_m % 16 == 0); + const n = weight.dim(1); + const k = input.dim(1); + std.debug.assert(supportsShape(n, k)); + std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and + scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and + output.dim(-1) == n); + // Keep packed-word counts and route/output indices within signed i32. + std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and + sorted_ids.count() <= std.math.maxInt(i32)); + const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); + std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and + input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); + const capacity: i64 = @intCast(sorted_ids.count()); + const blocks: i64 = @intCast(expert_ids.count()); + if (naive) { + std.debug.assert(blocks == routes); + } else { + std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and + blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); + } + if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); + return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); + } + + fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { + const n = weight.dim(1); + const k = input.dim(1); + const routes = input.dim(0) * @as(i64, @intCast(topk)); + const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; + const cfg: Config = .{ + .tokens = input.dim(0), + .experts = weight.dim(0), + .n = n, + .k = k, + .ka = @divExact(k, 4), + .kb = @divExact(k, 8), + .ks = @divExact(k, 32), + .topk = @as(i64, @intCast(topk)), + .routes = routes, + .capacity = @as(i64, @intCast(sorted_ids.count())), + .blocks = @as(i64, @intCast(expert_ids.count())), + .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), + .native_m = native_m, + .wide = @as(i64, @intFromBool(native_m == 32)), + .stage_elements = native_m * 128, + .shared_elements = native_m * 256, + .naive = @as(i64, @intFromBool(naive)), + }; + const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); + const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); + return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ + .output = zml.Shape.init(.{ routes, n }, .bf16), + }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, + }).output.reshape(output); + } + + const Config = struct { + tokens: i64, + experts: i64, + n: i64, + k: i64, + ka: i64, + kb: i64, + ks: i64, + topk: i64, + routes: i64, + capacity: i64, + blocks: i64, + subtiles: i64, + native_m: i64, + wide: i64, + stage_elements: i64, + shared_elements: i64, + naive: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "mxfp4_raw_mfma", + .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, + .outputs = &.{"output"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const zero = b.constant(.i32, 0); + const fzero = b.constant(.f32, 0.0); + const zeros = fzero.splat(4); + const block_n = b.blockId(.x).to(.i64); + const block_m = b.blockId(.y).to(.i64); + const route_start = block_m.mul(cfg.native_m); + const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { + const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); + var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{.i1}); + { + has_block.yieldThen(.{args.sorted.at(route_start).ne(cfg.routes)}); + } + { + has_block.yieldElse(.{b.constant(.i1, 0)}); + } + break :blk has_block.results[0]; + }; + var block = b.openIf(has_rows); + { + const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); + const eid = args.experts.at(expert_block); + const valid_expert = eid.ge(0).bitAnd(eid.lt(cfg.experts)); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(64); + const wave = thread.divUnsigned(64); + const mn_lane = lane.remUnsigned(16); + const k_lane = lane.divUnsigned(16); + const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); + // Each expert has a contiguous real prefix before route-count padding. + const second_active = if (cfg.wide != 0) blk: { + const first = args.sorted.at(route_start.add(16)); + break :blk first.ge(0).bitAnd(first.lt(cfg.routes)); + } else b.constant(.i1, 0); + + var sums = b.openIfElse(valid_expert, .{ zeros, zeros }); + { + const expert = eid.to(.i64); + const load_row = thread.divUnsigned(16); + const load_chunk = thread.remUnsigned(16); + const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); + const a_valid = a_route.ge(0).bitAnd(a_route.lt(cfg.routes)); + const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var second_route = b.openIfElse(second_active, .{zero}); + { + second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); + } + { + second_route.yieldElse(.{b.constant(.i32, -1)}); + } + const a_route_g1 = second_route.results[0]; + const a_valid_g1 = a_route_g1.ge(0).bitAnd(a_route_g1.lt(cfg.routes)); + const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + + // One allocation, two disjoint ping-pong regions; never two identical leaves. + const shared = b.sharedArray(.i16, cfg.shared_elements, 16); + const load_a = b.copyAtom(.{ .universal = 128 }, .i64); + const load_b = b.copyAtom(.{ .universal = 128 }, .i32); + const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); + const loader: TileLoader = .{ + .a = args.a, + .weight = args.b, + .scales = args.scales, + .shared = shared, + .load_a = load_a, + .load_b = load_b, + .copy_shared = copy_shared, + .a_base = a_base, + .a_base_g1 = a_base_g1, + .a_valid = a_valid, + .a_valid_g1 = a_valid_g1, + .second_active = second_active, + .a_col = load_chunk.mul(2), + .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), + .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), + .expert = expert, + .out_n = out_n, + .k_lane = k_lane, + }; + const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); + b.barrier(); + const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = fly.DType.bf16.toMlir(b.ctx), + .elemTyB = fly.DType.bf16.toMlir(b.ctx), + .elemTyAcc = fly.DType.f32.toMlir(b.ctx), + }) catch return error.InvalidMlir).type_()); + var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); + { + const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); + const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); + var a_fragments: [4]Value = undefined; + var a_fragments_g1: [4]Value = undefined; + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); + a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); + } + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); + const azero = b.constant(.i16, 0).splat(8); + var second = b.openIfElse(second_active, .{azero}); + { + second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); + } + { + second.yieldElse(.{azero}); + } + a_fragments_g1[u] = second.results[0]; + } + // Prefetch the other LDS region only after all current A fragments are read. + const next_k = k.iv.add(128); + var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words, initial.scale }); + { + const next_tile = loader.load(b, next_k, write_base); + next.yieldThen(.{ next_tile.words, next_tile.scale }); + } + { + next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); + } + var acc = k.carried[0]; + var acc_g1 = k.carried[1]; + inline for (0..4) |u| { + const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); + const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); + const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); + acc = b.mmaAtomCall(atom, a0, w0, acc); + acc = b.mmaAtomCall(atom, a1, w1, acc); + var second = b.openIfElse(second_active, .{zeros}); + { + const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); + second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); + } + { + second.yieldElse(.{acc_g1}); + } + acc_g1 = second.results[0]; + } + // All current reads finish before a later iteration reuses this LDS region. + b.barrier(); + k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); + } + sums.yieldThen(.{ k.results[0], k.results[1] }); + } + { + sums.yieldElse(.{ zeros, zeros }); + } + + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + } + var store_second = b.openIf(second_active); + { + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); + } + store_second.close(); + } + block.close(); + } + } + + fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { + return if (naive) row.eq(0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); + } + + fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { + const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); + var valid = b.openIf(route.ge(0).bitAnd(route.lt(cfg.routes))); + { + output.set(.{ route.to(.i64), col }, value.to(.bf16)); + valid.close(); + } + } + + fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { + return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); + } + + const TileLoader = struct { + a: Value, + weight: Value, + scales: Value, + shared: Value, + load_a: Value, + load_b: Value, + copy_shared: Value, + a_base: Value, + a_base_g1: Value, + a_valid: Value, + a_valid_g1: Value, + second_active: Value, + a_col: Value, + a_store: Value, + b_base: Value, + expert: Value, + out_n: Value, + k_lane: Value, + + fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { + self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); + var second = b.openIf(self.second_active); + { + self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); + second.close(); + } + const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); + const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); + const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); + const scale = byte.toUnsigned(.i32); + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return .{ .words = words, .scale = scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32) }; + } + + fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { + const zeros = b.constant(.i64, 0).splat(2); + var guarded = b.openIfElse(valid, .{zeros}); + { + const offset = base.add(k.divUnsigned(4)).add(self.a_col); + guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); + } + { + guarded.yieldElse(.{zeros}); + } + b.copyAtomStore(self.copy_shared, guarded.results[0].bitCast(.i16), vectorView(self.shared, destination, 8)); + } + }; + + fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { + const lo = b.constant(.i32, 0x44403800); + const hi = b.constant(.i32, 0x54504c48); + const even = b.perm(hi, lo, word.bitAnd(0x07070707)); + const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); + const ev0 = b.cvtPkF32Fp8(even, false); + const od0 = b.cvtPkF32Fp8(odd, false); + const ev1 = b.cvtPkF32Fp8(even, true); + const od1 = b.cvtPkF32Fp8(odd, true); + const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); + const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); + const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); + var shifts = b.constant(.i32, 0).splat(8); + inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); + const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); + // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. + return magnitude.bitCast(.i32).bitOr(signs).bitCast(.f32).mul(scale).to(.bf16); + } +}; + +const Gemv = struct { + // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. + // Several rows share a CTA, and the shorter down projection uses subwaves to + // expose independent outputs without paying for a full wave reduction per row. + const Config = struct { rows: usize, lanes: usize, unroll: usize }; + + /// One local expert ID per original route; negative/out-of-bank IDs write zero. + fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { + const cfg: Gemv.Config = if (x.dim(1) >= 4096) + .{ .rows = 4, .lanes = 64, .unroll = 1 } + else + .{ .rows = 4, .lanes = 32, .unroll = 5 }; + const rows = cfg.rows; + const lanes = cfg.lanes; + const unroll = cfg.unroll; + const n = weight.dim(1); + const k = x.dim(1); + const routes: i64 = @intCast(ids.count()); + std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); + std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); + std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); + std.debug.assert(weight.dim(2) * 2 == k); + std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); + const tile_rows = rows * (64 / lanes); + std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); + const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; + const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); + const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); + const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); + return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ + .output = matrix_output, + }, .{ + .cfg = fields, + .threads = @intCast(rows * 64), + .grid = .{ @intCast(fields.grid), 1, 1 }, + }).output.reshape(output); + } + + const EmitConfig = struct { + unroll: usize, + rows: usize, + lanes: usize, + stride: usize, + ntiles: i64, + tokens: i64, + experts: i64, + n: i64, + k_pairs: i64, + k_words: i64, + k_scales: i64, + routes: i64, + grid: i64, + topk: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ + .name = "mxfp4_gemv", + .inputs = &.{ "a", "b", "scales", "ids" }, + .outputs = &.{"output"}, + .run = Gemv.run, + }); + + fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { + const args = Gemv.Kernel.args(b); + const block = b.blockId(.x).to(.i64); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(cfg.lanes); + const rowgroup = thread.divUnsigned(cfg.lanes); + // Preserve the eight-XCD workgroup permutation and original subwave layout. + const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); + const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); + const route = pid.divUnsigned(cfg.ntiles); + const expert32 = args.ids.at(route); + const expert = expert32.to(.i64); + const token = route.divUnsigned(cfg.topk); + const local = expert32.ge(0).bitAnd(expert32.lt(cfg.experts)); + const inrow = row.cmpUnsigned(.lt, cfg.n); + const fzero = b.constant(.f32, 0.0); + var sum = b.openIfElse(local.bitAnd(inrow), .{fzero}); + { + var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); + { + var carried = k.carried[0]; + for (0..cfg.unroll) |u| { + const pos = k.iv.add(u * cfg.lanes); + var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero}); + { + const word = args.b.at(.{ expert, row, pos }); + const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); + const scale = decodeScale(b, scale8); + const low = word.bitAnd(0x07070707); + const high = word.shrU(4).bitAnd(0x07070707); + const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); + const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); + const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); + const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); + // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. + const weights = [_]Value{ + b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), + }; + var partial = fzero; + inline for (0..4) |pair| { + const x = args.a.at(.{ token, pos.mul(4).add(pair) }); + const xlo = x.to(.i16).bitCast(.bf16).to(.f32); + const xhi = x.shrU(16).to(.i16).bitCast(.bf16).to(.f32); + const wlo = weights[(pair / 2) * 2].extract(pair % 2); + const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); + // Odd then even FMA, followed by one add to the K-loop accumulator. + partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + } + inbounds.yieldThen(.{partial.add(carried)}); + } + { + inbounds.yieldElse(.{carried}); + } + carried = inbounds.results[0]; + } + k.yield(.{carried}); + } + var reduced = k.results[0]; + var shift: i32 = 1; + while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { + reduced = reduced.add(reduced.shuffleXor(shift, 64)); + } + sum.yieldThen(.{reduced}); + } + { + sum.yieldElse(.{fzero}); + } + var store = b.openIf(lane.eq(0).bitAnd(inrow)); + { + args.output.set(.{ route, row }, sum.results[0].to(.bf16)); + store.close(); + } + } + + fn decodeScale(b: *Builder, byte: Value) Value { + const scale = byte.toUnsigned(.i32); + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32); + } +}; diff --git a/zml/kernels/moe/metal/BUILD.bazel b/zml/kernels/moe/metal/BUILD.bazel new file mode 100644 index 00000000..b4240b00 --- /dev/null +++ b/zml/kernels/moe/metal/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + ], +) diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig new file mode 100644 index 00000000..f835c889 --- /dev/null +++ b/zml/kernels/moe/metal/metal.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; + +const log = std.log.scoped(.@"zml/moe/metal"); + +fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); +} + +const QuantMode = enum { none, fp8, nvfp4 }; + +fn quantMode(dtype: zml.DataType) QuantMode { + return switch (dtype) { + .bf16, .f16, .f32 => .none, + .f8e4m3fn => .fp8, + .u8, .f4e2m1 => .nvfp4, + else => stdx.debug.panic("unsupported data type: {}", .{dtype}), + }; +} + +fn moeGemm( + x_rows: Tensor, + w: Tensor, + scale: ?Tensor, + expert_ids: Tensor, + out_shape: zml.Shape, + mode: QuantMode, +) Tensor { + const side: zml.ops.CustomCallOptions = .{ .has_side_effect = false }; + return switch (mode) { + .none => zml.ops.customCall("__metal$moe_gemm", .{ x_rows, w, expert_ids }, .{out_shape}, .{}, side), + .fp8 => zml.ops.customCall("__metal$moe_gemm$f8", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + .nvfp4 => zml.ops.customCall("__metal$moe_gemm$f4", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + }; +} + +fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { + const scale = global_scale orelse return output; + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected gate_up scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 2 and scale.dim(.proj) == 2, "metal backend expected gate_up scale to be a 2D tensor with a .proj dim equal to 2, got {}", .{scale.shape()}); + + const mid = @divExact(output.dim(.out), 2); + const selected = scale.gather(.{ .expert = expert_ids }, .{}); + const expanded = selected.appendAxes(.{.mid}) + .broad(zml.Shape.init(.{ .r = output.dim(.r), .proj = 2, .mid = mid }, .f32)) + .merge(.{ .out = .{ .proj, .mid } }); + return output.convert(.f32).mul(expanded).convert(output.dtype()); +} + +fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { + const scale = global_scale orelse return output; + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected down scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 1, "metal backend expected down scale to be a 1D tensor, got {}", .{scale.shape()}); + + const selected = scale.gather(.{ .expert = expert_ids }, .{}) + .appendAxes(.{.d}) + .broad(output.shape().withDtype(.f32)); + return output.convert(.f32).mul(selected).convert(output.dtype()); +} + +pub fn fusedExperts( + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + validateOptions(opts); + stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); + + const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); + stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); + const mode = quantMode(gate_up.dtype()); + switch (mode) { + .none => { + stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); + }, + .fp8 => { + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); + }, + .nvfp4 => { + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); + }, + } + + const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); + stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); + + const d = hidden_states.dim(.d); + const num_tokens = hidden_states.dim(.t); + const topk = routing.ids.dim(.topk); + const num_routes = num_tokens * topk; + + const act_dtype: zml.DataType = switch (mode) { + .none => hidden_states.dtype(), + .fp8, .nvfp4 => .bf16, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); + const x_rows = hidden.insertAxes(.d, .{.topk}) + .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) + .merge(.{ .r = .{ .token, .topk } }); + const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }).convert(.i32); + + const gate_up_out = applyGateUpGlobalScale(moeGemm( + x_rows, + gate_up, + gate_up_scales, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), + mode, + ), gate_up_global_scale, expert_ids); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); + + const down_out = applyDownGlobalScale(moeGemm( + activated, + down, + down_scales, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), + mode, + ), down_global_scale, expert_ids); + + const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }); + const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); + const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) + .sum(.topk).squeeze(.topk); + + return combined.reshape(.{ .t = num_tokens, .d = down.dim(.d) }); +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig new file mode 100644 index 00000000..eb697f36 --- /dev/null +++ b/zml/kernels/moe/moe.zig @@ -0,0 +1,298 @@ +const std = @import("std"); + +const platforms = @import("platforms"); + +const zml = @import("../../zml.zig"); +const stdx = zml.stdx; +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; +pub const cute = @import("cute/cute.zig"); +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); +pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +const log = std.log.scoped(.@"zml/moe"); + +test { + std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); +} + +pub const Backend = enum { + stablehlo, + cute, + flashinfer_cutlass, + triton, + fly, + mosaic_tpu, + metal, + tcl, + + pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + return switch (platform.target) { + .cuda => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (cute.isAvailable(platform)) + .cute + else + .triton, + .nvfp4 => if (cute.isNvfp4Supported(platform)) + .cute + else if (flashinfer_cutlass.isNvfp4Supported(platform)) + .flashinfer_cutlass + else + error.UnsupportedQuantization, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + }; + }, + .rocm => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (zml.platform.rocm.computeCapability(platform) == .gfx942) .fly else .triton, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + .nvfp4 => error.UnsupportedQuantization, + }; + }, + .oneapi => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => .triton, + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => error.UnsupportedQuantization, + }; + }, + .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, + .furiosa => b: { + const s = scheme orelse break :b .tcl; + break :b switch (s) { + .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, + .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, + }; + }, + .metal => b: { + const s = scheme orelse break :b .metal; + break :b switch (s) { + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .metal, + .mxfp4 => error.UnsupportedQuantization, + }; + }, + else => error.UnimplementedMoEBackend, + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .cute => cute.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), + .fly => switch (platform.target) { + .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, + else => false, + }, + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + else => false, + }, + .mosaic_tpu => platform.target == .tpu, + .metal => platform.target == .metal, + .tcl => platform.target == .furiosa, + }; + } + + /// The layout contract the expert weights must have for a given backend with scheme + pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { + return switch (backend) { + .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, + .cute => if (scheme) |s| switch (s) { + .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, + .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, + .flashinfer_cutlass => if (scheme) |s| switch (s) { + .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, + .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + }; + } + + pub fn register(backend: Backend, platform: *zml.Platform) !void { + return switch (backend) { + .stablehlo => {}, + .cute => {}, + .flashinfer_cutlass => flashinfer_cutlass.register(platform), + .triton, .fly => {}, + .mosaic_tpu => {}, + .metal => {}, + .tcl => {}, + }; + } +}; + +/// Check the shared token-wise contract for activations and routing tensors. +pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { + stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); + stdx.debug.assert(input.dim(.t) == ids.dim(.t) and input.dim(.t) == weights.dim(.t), "MoE input and routing token counts must match", .{}); + stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "MoE routing ids and weights must have matching top-k dimensions", .{}); + stdx.debug.assert(input.shape().hasTag(.t) == 0 and input.shape().hasTag(.d) == 1 and + ids.shape().hasTag(.t) == 0 and ids.shape().hasTag(.topk) == 1 and + weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); +} + +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + input: Logical, + ids: Logical, + weights: Logical, + gate_up: Logical, + down: Logical, + }, + out_specs: Logical, + + /// Shards the `.expert` dimension of both projections over `mesh_axis`. + /// Other dimensions, input tokens, routing IDs/weights and output are replicated. + pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { + const experts: Logical = .parse(.{ .expert = mesh_axis }); + return .{ + .in_specs = .{ + .input = .replicated, + .ids = .replicated, + .weights = .replicated, + .gate_up = experts, + .down = experts, + }, + .out_specs = .replicated, + }; + } +}; + +/// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. +/// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). +pub fn forwardMoe( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + mesh: *const zml.Mesh, + partitioning: Partitioning, + opts: Options, +) zml.Tensor { + validateRoutingInputs(input, topk_ids, topk_weights); + + const shapes = .{ + .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), + .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), + .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), + .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), + .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), + }; + const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); + + const tokens = shapes.input.partition(.t); + const experts = shapes.gate_up.partition(.expert); + // Local kernels require complete feature vectors and top-k routing lists. + stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); + // The expert-axis reduction must add contributions for the same token rows. + // Sharding tokens along that axis would mix contributions for different tokens. + stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); + stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); + for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { + stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); + for (0..shape.rank()) |axis| { + stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); + } + } + const expert_axis = experts.meshAxis(); + const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; + if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + + const Call = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + opts: Options, + mesh: *const zml.Mesh, + expert_axis: ?u3, + global_experts: u32, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + const routing: LocalRouting = .{ + .ids = args.ids, + .weights = args.weights, + .global_num_experts = args.global_experts, + .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), + .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, + }; + const output = switch (args.backend) { + .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + }; + return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; + } + }; + const args: Call = .{ + .input = input, + .ids = topk_ids, + .weights = topk_weights, + .gate_up = gate_up, + .down = down, + .backend = backend, + .opts = opts, + .mesh = mesh, + .expert_axis = if (parallel) expert_axis else null, + .global_experts = @intCast(gate_up.weight.dim(.expert)), + }; + const Logical = zml.Sharding.Partitioning.Logical; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; + for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + zml.meta.mapAlloc(struct { + fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { + const axis = ctx.axis orelse return .replicated; + if (tensor.count() == 1) return .replicated; + // Packed scales may be untagged; experts are their leading dimension. + const dim = tensor.shape().hasTag(.expert) orelse 0; + stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); + return .{ .mapping = .{ + .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), + .tgt = .init(&.{axis}), + } }; + } + }.spec, zml.Compiler.current().arena.allocator(), .{ + .experts = linear.weight.dim(.expert), + .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, + }, linear, specs) catch @panic("OOM"); + } + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ + .in_specs = .{ + .input = partitioning.in_specs.input, + .ids = partitioning.in_specs.ids, + .weights = partitioning.in_specs.weights, + .gate_up = projection_specs[0], + .down = projection_specs[1], + }, + .out_specs = partitioning.out_specs, + }); +} diff --git a/zml/kernels/moe/mosaic_tpu/BUILD.bazel b/zml/kernels/moe/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ed4dfcd9 --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "gmm_ep.zig", + "mosaic_tpu.zig", + ], +) diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig new file mode 100644 index 00000000..c30d805b --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -0,0 +1,573 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const mtt = @import("dsl/mosaic_tpu/builder"); +const pipeline = mtt.pipeline; +const mtt_kernel = zml.kernel.mosaic_tpu; + +fn cdiv(x: i64, y: i64) i64 { + return @divFloor(x + y - 1, y); +} + +fn alignTo(x: i64, y: i64) i64 { + return cdiv(x, y) * y; +} + +fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.constant(.i32, 0); + const r = b.remsi(x, y); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + return b.select(b.andi(r_neg, r_nz), b.addi(r, y), r); +} + +fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + const q = b.divsi(x, y); + const sign = b.subi( + b.extui(b.cmpi(.sgt, x, c0), .i32), + b.extui(b.cmpi(.slt, x, c0), .i32), + ); + const sign_not_pos = b.cmpi(.ne, sign, c1); + const rem_nz = b.cmpi(.ne, b.remsi(x, y), c0); + return b.select(b.andi(sign_not_pos, rem_nz), b.subi(q, c1), q); +} + +fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + return b.divsi(b.addi(x, b.subi(y, b.constant(.i32, 1))), y); +} + +pub const Cfg = struct { + pub const FuseAct = enum { + none, + silu, + gelu, + }; + + size_m: i64, + size_k: i64, + size_n: i64, + size_group: i64, + size_lhs_group: i64, + tile_m: i64, + tile_k: i64, + tile_n: i64, + lhs_dtype: mtt.DType = .bf16, + rhs_dtype: mtt.DType = .bf16, + out_dtype: mtt.DType = .bf16, + acc_dtype: mtt.DType = .f32, + size_lhs_sublane: i64 = 8, + fuse_act: FuseAct = .none, + + pub fn outSizeN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.size_n, + .silu, .gelu => @divExact(self.size_n, 2), + }; + } + + pub fn alignedN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => alignTo(self.outSizeN(), 128), + .silu, .gelu => self.outSizeN(), + }; + } + + pub fn rhsTileN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.tile_n, + .silu, .gelu => self.tile_n * 2, + }; + } + + pub fn maxNumGm(self: Cfg) i64 { + return self.size_group + cdiv(self.size_m, self.tile_m) - 1; + } +}; + +fn validateConfig(cfg: Cfg) void { + std.debug.assert(cfg.size_group <= cfg.size_lhs_group); + std.debug.assert(cfg.size_m > 0); + std.debug.assert(cfg.size_k > 0); + std.debug.assert(cfg.size_n > 0); + std.debug.assert(cfg.tile_m > 0); + std.debug.assert(cfg.tile_k > 0); + std.debug.assert(cfg.tile_n > 0); + std.debug.assert(cfg.size_lhs_sublane > 0); + std.debug.assert(@mod(cfg.size_m, cfg.size_lhs_sublane) == 0); + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cfg.tile_k, 128) == 0); + std.debug.assert(@mod(cfg.rhsTileN(), 128) == 0); + if (cfg.fuse_act != .none) { + std.debug.assert(@mod(cfg.size_n, 2) == 0); + std.debug.assert(cdiv(cfg.alignedN(), cfg.tile_n) == 1); + } else { + std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); + } + std.debug.assert(cfg.acc_dtype == .f32); +} + +const Meta = struct { + group_id: mtt.Value, + m_offset: mtt.Value, +}; + +const PipeCtx = struct { + cfg: Cfg, + meta: Meta, + num_gm: mtt.Value, +}; + +fn winRef(b: *mtt.Builder, rw: pipeline.RefWindow, shape: []const i64) mtt.Value { + const buf = rw.buf.?; + const slot = rw.slot orelse return buf; + const a = b.arena.allocator(); + const rank = shape.len + 1; + const base = a.alloc(mtt.Value, rank) catch @panic("gmm_ep winRef OOM"); + const result_shape = a.alloc(i64, rank) catch @panic("gmm_ep winRef OOM"); + base[0] = slot; + result_shape[0] = 1; + for (shape, 0..) |d, i| { + base[i + 1] = b.constant(.i32, 0); + result_shape[i + 1] = d; + } + const sliced = b.memRefSlice(buf, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, shape); +} + +fn metaLoad(b: *mtt.Builder, ref: mtt.Value, idx: mtt.Value) mtt.Value { + return b.scalarLoad(ref, &.{b.toIndex(idx)}); +} + +fn lhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const k_id = indices[2]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const row_start = floorDivPosI32(b, m_start, sublane); + const row_end = ceilDivPosI32(b, m_end, sublane); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep lhsIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; + out[2] = .{ .scalar = k_id }; + return out; +} + +fn rhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const group_id = metaLoad(b, ctx.meta.group_id, indices[1]); + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep rhsIndexMap OOM"); + out[0] = .{ .scalar = group_id }; + out[1] = .{ .scalar = indices[2] }; + out[2] = .{ .scalar = indices[0] }; + return out; +} + +fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const n_id = indices[0]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const row_start = floorDivPosI32(b, m_start, sublane); + const capped_row_end = floorDivPosI32(b, m_end, sublane); + const last_row_end = ceilDivPosI32(b, m_end, sublane); + const row_end = b.select( + b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.constant(.i32, 1))), + last_row_end, + capped_row_end, + ); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep outIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; + out[2] = .{ .scalar = n_id }; + return out; +} + +fn indices1DTo3D(b: *mtt.Builder, indices: []const mtt.Value) []const mtt.Value { + const out = b.arena.allocator().alloc(mtt.Value, 3) catch @panic("gmm_ep indices1DTo3D OOM"); + out[0] = b.constant(.i32, 0); + out[1] = indices[0]; + out[2] = b.constant(.i32, 0); + return out; +} + +fn lhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return lhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn rhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return rhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn outIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return outIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + const group_offset = b.scalarLoad(a.group_offset, &.{b.constIndex(0)}); + const max_num_group = b.addi(group_offset, b.constant(.i32, cfg.size_group)); + b.scalarStore(a.metadata_m_offset, c0, &.{b.constIndex(0)}); + + var outer = b.openFor(c0, max_num_group, c1, .{ c0, c0 }); + { + b.traceStart(10, "outer_group_loop"); + const lhs_group_id = outer.iv; + const num_gm = outer.carried[0]; + const start_m_offset = outer.carried[1]; + const group_id = b.subi(lhs_group_id, group_offset); + const group_size = b.scalarLoad(a.group_sizes, &.{b.toIndex(lhs_group_id)}); + const end_m_offset = b.addi(start_m_offset, group_size); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const local_offset = floorModI32(b, start_m_offset, sublane); + const aligned_group_size = b.addi(group_size, local_offset); + const tile_m = b.constant(.i32, cfg.tile_m); + var curr_num_gm = b.divsi(b.addi(aligned_group_size, b.subi(tile_m, c1)), tile_m); + const should_process = b.andi(b.cmpi(.sgt, group_size, c0), b.cmpi(.sge, group_id, c0)); + curr_num_gm = b.select(should_process, curr_num_gm, c0); + const next_num_gm = b.addi(num_gm, curr_num_gm); + + var inner = b.openFor(num_gm, next_num_gm, c1, .{start_m_offset}); + { + b.traceStart(10, "inner_tm_loop"); + const tm_id = inner.iv; + const curr_m_offset = inner.carried[0]; + const local = floorModI32(b, curr_m_offset, sublane); + const tm_room = b.subi(tile_m, local); + const remaining = b.subi(end_m_offset, curr_m_offset); + const tm_size = b.minsi(tm_room, remaining); + const next_m_offset = b.addi(curr_m_offset, tm_size); + b.scalarStore(a.metadata_group_id, group_id, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, curr_m_offset, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, next_m_offset, &.{b.toIndex(b.addi(tm_id, c1))}); + b.traceStop(); + inner.yield(.{next_m_offset}); + } + + b.traceStop(); + outer.yield(.{ next_num_gm, end_m_offset }); + } + return outer.results[0]; +} + +fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, cols: i64) mtt.Value { + if (cfg.tile_k > cfg.size_k) { + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cols, 128) == 0); + + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + const tail_k = cfg.tile_k - cfg.size_k; + const lhs_valid = b.vectorExtractStridedSlice(lhs, &.{ 0, 0 }, &.{ cfg.tile_m, cfg.size_k }, &.{ 1, 1 }); + const lhs_tail = b.vectorExtractStridedSlice(lhs, &.{ 0, cfg.size_k }, &.{ cfg.tile_m, tail_k }, &.{ 1, 1 }); + + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_valid = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.size_k, 128 }, &.{ 1, 1 }); + const rhs_tail = b.vectorExtractStridedSlice(rhs, &.{ cfg.size_k, start_n }, &.{ tail_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + const tail_acc = b.matmulOpts(lhs_tail, rhs_tail, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + parts[@intCast(chunk)] = b.matmulOpts(lhs_valid, rhs_valid, tail_acc, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + if (chunks == 1) return parts[0]; + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); + } + + if (cols <= 128) { + return b.matmulOpts(lhs, rhs, b.zeros(&.{ cfg.tile_m, cols }, cfg.acc_dtype), .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + std.debug.assert(@mod(cols, 128) == 0); + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_chunk = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.tile_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + parts[@intCast(chunk)] = b.matmulOpts(lhs, rhs_chunk, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + if (chunks == 1) return parts[0]; + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); +} + +fn geluApprox(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const x2 = b.mulf(x, x); + const x3 = b.mulf(x2, x); + const cubic = b.mulf(x3, b.splat(@as(f32, 0.044715), shape, cfgAccDtype())); + const inner = b.mulf(b.addf(x, cubic), b.splat(@as(f32, 0.7978845608028654), shape, cfgAccDtype())); + return b.mulf( + b.mulf(x, b.splat(@as(f32, 0.5), shape, cfgAccDtype())), + b.addf(b.tanh(inner), b.splat(@as(f32, 1.0), shape, cfgAccDtype())), + ); +} + +fn cfgAccDtype() mtt.DType { + return .f32; +} + +fn silu(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const one = b.splat(@as(f32, 1.0), shape, cfgAccDtype()); + const denom = b.addf(one, b.exp(b.negf(x))); + return b.divf(x, denom); +} + +fn applyFuseAct(b: *mtt.Builder, cfg: Cfg, acc: mtt.Value) mtt.Value { + return switch (cfg.fuse_act) { + .none => acc, + .silu, .gelu => { + const shape = &.{ cfg.tile_m, cfg.tile_n }; + const gate = b.vectorExtractStridedSlice(acc, &.{ 0, 0 }, shape, &.{ 1, 1 }); + const up = b.vectorExtractStridedSlice(acc, &.{ 0, cfg.outSizeN() }, shape, &.{ 1, 1 }); + const activated = switch (cfg.fuse_act) { + .silu => silu(b, gate, shape), + .gelu => geluApprox(b, gate, shape), + .none => unreachable, + }; + return b.mulf(activated, up); + }, + }; +} + +fn innerBody( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + + const lhs_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_k }; + const out_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }; + const rhs_tile_n = cfg.rhsTileN(); + + b.traceStart(10, "matmul_first_last"); + const lhs_ref = winRef(b, refs[0], lhs_shape); + const lhs_ref_2d = b.memRefReshape(lhs_ref, &.{ cfg.tile_m, cfg.tile_k }); + const lhs = b.refLoad(lhs_ref_2d); + const rhs_slot = b.toIndex(refs[1].slot.?); + var rhs = b.shapeCast( + b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), + &.{ cfg.tile_k, rhs_tile_n }, + ); + const valid_k = @mod(cfg.size_k, cfg.tile_k); + if (valid_k != 0) { + const k_iota = b.iota(&.{ cfg.tile_k, rhs_tile_n }, .i32, &.{0}); + const valid = b.splat(@as(i32, @intCast(valid_k)), &.{ cfg.tile_k, rhs_tile_n }, .i32); + rhs = b.select(b.cmpi(.slt, k_iota, valid), rhs, b.zeros(&.{ cfg.tile_k, rhs_tile_n }, cfg.rhs_dtype)); + } + var acc = matmulByMxuColumns(b, cfg, lhs, rhs, rhs_tile_n); + acc = applyFuseAct(b, cfg, acc); + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const m_offset = b.subi(m_start, floorModI32(b, m_start, sublane)); + const m_start_local = b.subi(m_start, m_offset); + const m_end_local = b.subi(m_end, m_offset); + const row_iota = b.iota(&.{ cfg.tile_m, cfg.tile_n }, .i32, &.{0}); + const row_mask = b.andi( + b.cmpi(.sle, b.broadcastTo(m_start_local, &.{ cfg.tile_m, cfg.tile_n }), row_iota), + b.cmpi(.slt, row_iota, b.broadcastTo(m_end_local, &.{ cfg.tile_m, cfg.tile_n })), + ); + acc = b.select(row_mask, acc, b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.acc_dtype)); + + const out_slot_i32 = refs[2].slot.?; + const out_slot = b.toIndex(out_slot_i32); + b.vectorStoreAt( + refs[2].buf.?, + b.shapeCast(b.shapeCast(acc, out_shape).to(cfg.out_dtype), &.{ 1, cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }), + &.{ out_slot, b.constIndex(0), b.constIndex(0), b.constIndex(0) }, + ); + + const out_ref = winRef(b, refs[2], out_shape); + if (cfg.tile_m == cfg.size_lhs_sublane) { + const out_value = b.shapeCast(b.refLoad(out_ref), &.{ cfg.tile_m, cfg.tile_n }); + const partial_zero = b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(out_value, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.refStore(out_ref_for_store, b.shapeCast(out_accum, out_shape)); + } else { + const first_row = b.shapeCast( + b.vectorLoadShape(out_ref, &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ cfg.size_lhs_sublane, cfg.tile_n }, + ); + const partial_zero = b.zeros(&.{ cfg.size_lhs_sublane, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(first_row, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.vectorStoreAt( + out_ref_for_store, + b.shapeCast(out_accum, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, + ); + } + + const last_row = floorDivPosI32(b, m_end_local, sublane); + const m_end_mod = floorModI32(b, m_end_local, sublane); + const partial_complete = b.cmpi(.eq, m_end_mod, b.constant(.i32, 0)); + const out_ref_for_partial = winRef(b, refs[2], out_shape); + const partial_shape = if (cfg.tile_m == cfg.size_lhs_sublane) + &.{ cfg.tile_m, cfg.tile_n } + else + &.{ cfg.size_lhs_sublane, cfg.tile_n }; + const partial_zero = b.zeros(partial_shape, cfg.out_dtype); + const partial_row = b.shapeCast( + b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + partial_shape, + ); + b.refStore(scratches[0], b.select(partial_complete, partial_zero, partial_row)); + b.traceStop(); +} + +fn innerBody1D( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + try innerBody(b, indices1DTo3D(b, indices), refs, scratches, ctx_opaque); +} + +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + validateConfig(cfg); + + const a = try b.declareArgsOpts(.{ + .group_sizes = .{ .ref = .{ .shape = &.{cfg.size_lhs_group}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .group_offset = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .lhs = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.size_k }, .dtype = cfg.lhs_dtype, .memory_space = .hbm } }, + .rhs = .{ .ref = .{ .shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, .dtype = cfg.rhs_dtype, .memory_space = .hbm } }, + .out = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.alignedN() }, .dtype = cfg.out_dtype, .memory_space = .hbm, .role = .output } }, + .partial_out = .{ .ref = .{ .shape = &.{ cfg.size_lhs_sublane, cfg.tile_n }, .dtype = cfg.out_dtype, .role = .scratch } }, + .acc = .{ .ref = .{ .shape = &.{ cfg.tile_m, cfg.tile_n }, .dtype = cfg.acc_dtype, .role = .scratch } }, + .metadata_group_id = .{ .ref = .{ .shape = &.{cfg.maxNumGm()}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + .metadata_m_offset = .{ .ref = .{ .shape = &.{cfg.maxNumGm() + 1}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{}, + .scalar_prefetch = 2, + .scratch_operands = 4, + }); + + const num_k = b.constant(.i32, cdiv(cfg.size_k, cfg.tile_k)); + const num_n = b.constant(.i32, cdiv(cfg.alignedN(), cfg.tile_n)); + const num_gm = fillMetadata(b, cfg, a); + + const ctx = b.arena.allocator().create(PipeCtx) catch @panic("gmm_ep PipeCtx OOM"); + ctx.* = .{ + .cfg = cfg, + .meta = .{ .group_id = a.metadata_group_id, .m_offset = a.metadata_m_offset }, + .num_gm = num_gm, + }; + + const lhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .dtype = cfg.lhs_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_k } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) lhsIndexMap1D else lhsIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const rhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, + .dtype = cfg.rhs_dtype, + .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.rhsTileN() } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) rhsIndexMap1D else rhsIndexMap, + .index_map_ctx = ctx, + .buffer_count = 3, + }; + const out_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .dtype = cfg.out_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_n } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) outIndexMap1D else outIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const use_1d_pipeline = cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m; + try pipeline.emitPipeline(b, &.{ a.lhs, a.rhs, a.out }, &.{ a.partial_out, a.acc, a.metadata_group_id, a.metadata_m_offset }, .{ + .body = if (use_1d_pipeline) innerBody1D else innerBody, + .body_ctx = ctx, + .grid = if (use_1d_pipeline) &.{num_gm} else &.{ num_n, num_gm, num_k }, + .in_specs = &.{ lhs_spec, rhs_spec }, + .out_specs = &.{out_spec}, + }); +} + +pub const TpuKernel = mtt_kernel.Kernel(Cfg, .{ + .name = "gmm_ep_kernel", + .inputs = &.{ "group_sizes", "group_offset", "lhs", "rhs" }, + .outputs = &.{"out"}, + .run = run, +}); diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..bb60594b --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,301 @@ +const std = @import("std"); + +const mtt = @import("dsl/mosaic_tpu/builder"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +pub const gmm_ep = @import("gmm_ep.zig"); + +const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); + +var gate_up_transpose: std.atomic.Value(bool) = .init(false); +var down_transpose: std.atomic.Value(bool) = .init(false); + +fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); +} + +fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { + switch (hidden.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (gate_up.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (down.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (weights.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + if (ids.dtype() != .i32) return error.UnsupportedType; + if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; + if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; + if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; + if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; + if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; + if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; +} + +fn warningTranspose(flag: *std.atomic.Value(bool), projection_name: []const u8, expected_shape: []const u8) void { + if (!flag.swap(true, .monotonic)) { + log.warn("{s} MoE weights are transposed at runtime; weights must be loaded with the correct shape {s} to avoid the transpose at runtime.", .{ projection_name, expected_shape }); + } +} + +fn canonicalizeGateUp(w1: Tensor, hidden_size: i64) !Tensor { + if (w1.rank() != 3) return error.InvalidShape; + + if (w1.dim(1) == hidden_size) return w1.withTags(.{ .expert, .in, .out }); + if (w1.dim(2) == hidden_size) { + warningTranspose(&gate_up_transpose, "gate_up_proj", "[expert, in, out]"); + return w1.withTags(.{ .expert, .out, .in }).transpose(.{ .expert, .in, .out }); + } + return error.InvalidShape; +} + +fn canonicalizeDown(w2: Tensor, hidden_size: i64, gate_up_out: i64) !Tensor { + if (w2.rank() != 3) return error.InvalidShape; + + const mid_size = @divFloor(gate_up_out, 2); + if (w2.dim(1) == mid_size and w2.dim(2) == hidden_size) return w2.withTags(.{ .expert, .mid, .out }); + if (w2.dim(1) == hidden_size and w2.dim(2) == mid_size) { + warningTranspose(&down_transpose, "down_proj", "[expert, mid, out]"); + return w2.withTags(.{ .expert, .out, .mid }).transpose(.{ .expert, .mid, .out }); + } + return error.InvalidShape; +} + +fn histogramCounts(ids: Tensor, num_bins: i64) Tensor { + const zeros = Tensor.zeroes(.init(.{ .expert = num_bins }, .i32)); + const ones = Tensor.constant(.{ .i32 = 1 }).broad(ids.shape().withDtype(.i32)); + return zeros.scatterSlices( + .{ .expert = ids }, + ones, + .{ + .update_fn = Tensor.ScatterOpts.increment, + .indices_are_unique = false, + }, + ); +} + +fn alignSortedRowsByGroup(rows: Tensor, expert_ids_sorted: Tensor, group_sizes: Tensor, tile_m: i64) struct { + rows: Tensor, + group_sizes: Tensor, + positions: Tensor, +} { + const num_groups = group_sizes.dim(.expert); + const max_padded_rows = rows.dim(.token) + num_groups * (tile_m - 1); + + const group_ends = group_sizes.cumulativeSum(.expert); + const group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + group_ends.slice(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const padded_group_sizes = group_sizes.addConstant(tile_m - 1).divByConst(tile_m).mul(Tensor.scalar(tile_m, .i32)); + const padded_group_ends = padded_group_sizes.cumulativeSum(.expert); + const padded_group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + padded_group_ends.slice(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const logical_positions = Tensor.arange(.{ .end = rows.dim(.token) }, .i32).withTags(.{.token}); + const group_start_per_token = group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_group_start_per_token = padded_group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_positions = logical_positions.sub(group_start_per_token).add(padded_group_start_per_token).withTags(.{.token}); + + const padded = Tensor.zeroes(rows.shape().setDim(.token, max_padded_rows)) + .scatterSlices( + .{ .token = padded_positions }, + rows, + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + + return .{ + .rows = padded, + .group_sizes = padded_group_sizes, + .positions = padded_positions, + }; +} + +fn permuteSmallBatch(hidden: Tensor, sorted_route: Tensor, topk: i64) Tensor { + const num_tokens = hidden.dim(.token); + const num_routes = num_tokens * topk; + + const token_indices = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token}) + .repeat1d(.token, @intCast(topk)) + .rename(.{ .token = .route }); + const token_indices_sorted = token_indices.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .route }); + + const onehot_shape = zml.Shape.init(.{ .route = num_routes, .token_src = num_tokens }, .i32); + const token_indices_2d = token_indices_sorted.insertAxes(.last, .{.token_src}).broad(onehot_shape); + const source_tokens_2d = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token_src}) + .insertAxes(0, .{.route}) + .broad(onehot_shape); + const onehot = token_indices_2d.cmp(.EQ, source_tokens_2d).convert(hidden.dtype()); + + return onehot.dot(hidden.rename(.{ .token = .token_src }), .token_src).rename(.{ .route = .token }); +} + +fn gmmDType(dtype: zml.DataType) mtt.DType { + return switch (zml.kernel.mosaic_tpu.from(dtype)) { + .bf16 => .bf16, + .f16 => .f16, + .f32 => .f32, + else => stdx.debug.panic("mosaic_tpu MoE expects bf16/f16/f32 tensors, got {}", .{dtype}), + }; +} + +pub fn callGmmEp( + lhs: Tensor, + rhs_k_n: Tensor, + group_sizes: Tensor, + out_dtype: zml.DataType, + fuse_act: gmm_ep.Cfg.FuseAct, +) Tensor { + const k = lhs.dim(1); + const n = rhs_k_n.dim(.out); + const out_n = switch (fuse_act) { + .none => n, + .silu, .gelu => @divExact(n, 2), + }; + const aligned_k = @divFloor(k + 127, 128) * 128; + const aligned_n = if (fuse_act == .none) @divFloor(out_n + 127, 128) * 128 else out_n; + + const cfg: gmm_ep.Cfg = .{ + .size_m = lhs.dim(.token), + .size_k = k, + .size_n = n, + .size_group = rhs_k_n.dim(.expert), + .size_lhs_group = group_sizes.dim(.expert), + .tile_m = 128, + .tile_k = aligned_k, + .tile_n = if (fuse_act == .none) aligned_n else out_n, + .lhs_dtype = gmmDType(lhs.dtype()), + .rhs_dtype = gmmDType(rhs_k_n.dtype()), + .out_dtype = gmmDType(out_dtype), + .acc_dtype = .f32, + .size_lhs_sublane = 8, + .fuse_act = fuse_act, + }; + + const out = gmm_ep.TpuKernel.call( + .{ + .group_sizes = group_sizes.withTags(.{.expert}), + .group_offset = Tensor.zeroes(.init(.{1}, .i32)), + .lhs = lhs, + .rhs = rhs_k_n, + }, + .{ + .out = zml.Shape.init(.{ .token = lhs.dim(.token), .out = aligned_n }, out_dtype), + }, + .{ + .cfg = cfg, + .extras = .{ + // todo leverage tpu api like this https://github.com/openxla/tokamax/blob/f65135e85acd7571531be91d4d3241e1929421f0/tokamax/_src/ops/ragged_dot/pallas_mosaic_tpu_v2_tgmm_kernel.py#L692 + .vmem_limit_bytes = 32 * 1024 * 1024, + }, + }, + ).out; + + return out.slice(.out, .{ .end = out_n }); +} + +pub fn fusedExperts( + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + validateOptions(opts); + const expert_map_ = routing.expertMap(); + + const tokens = hidden_states.dim(.t); + const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); + // TODO(Corentin): Better error message + const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); + const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); + const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); + const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); + + const num_experts = routing.global_num_experts; + if (expert_map_) |expert_map| { + if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); + if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); + } + + const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + + const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { + const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); + const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); + const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); + break :blk .{ safe_ids, route_mask }; + } else .{ + flat_ids_global, + Tensor.scalar(true, .bool).broad(flat_ids_global.shape()), + }; + + const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); + const hidden_sorted = if (expert_map_ != null) + permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) + else blk: { + // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route + const hidden_repeated = hidden.repeat1d(.token, @intCast(ids.dim(.topk))).rename(.{ .token = .route }); + break :blk hidden_repeated.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + }; + const expert_ids_sorted = flat_ids_local.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const local_route_mask_sorted = local_route_mask.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const sorted_weights = flat_weights.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const router_weights_sorted = local_route_mask_sorted.select( + sorted_weights, + Tensor.zeroes(sorted_weights.shape()), + ); + + const group_sizes = histogramCounts(flat_ids_local, gate_up.dim(.expert)); + const tile_m: i64 = 128; + const aligned_hidden = alignSortedRowsByGroup(hidden_sorted, expert_ids_sorted, group_sizes, tile_m); + + var gate_up_out = callGmmEp(aligned_hidden.rows, gate_up, aligned_hidden.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (gate_up_.bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + gate_up_out = gate_up_out.add(bias_per_token); + } + + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); + + const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); + var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (down_.bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + down_out = down_out.add(bias_per_token); + } + + const weighted = down_out.mul(router_weights_sorted.convert(down_out.dtype()).broad(down_out.shape())); + + const restore_indices = sorted_route.argsort(.perm, .{ .descending = false }).rename(.{ .perm = .restore }); + const unsorted = weighted.gather(.{ .token = restore_indices }, .{}).rename(.{ .restore = .route }); + const restored = unsorted.reshape(.{ .topk = ids.dim(.topk), .token = hidden.dim(.token), .out = down.dim(.out) }); + const combined = restored.sum(.topk).squeeze(.topk); + return combined.reshape(.{ .t = tokens, .d = down.dim(.out) }); +} diff --git a/zml/kernels/moe/stablehlo/BUILD.bazel b/zml/kernels/moe/stablehlo/BUILD.bazel new file mode 100644 index 00000000..c31b27f0 --- /dev/null +++ b/zml/kernels/moe/stablehlo/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = [ + "stablehlo.zig", + ], +) diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig new file mode 100644 index 00000000..ba723234 --- /dev/null +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -0,0 +1,225 @@ +//! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. +//! Weights use [expert, output, input] order with concatenated gate/up columns. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; + +pub fn activate(x: Tensor, activation: Activation) Tensor { + return switch (activation) { + .silu => x.silu(), + .gelu => x.gelu(), + .relu => x.relu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => blk: { + const mid = @divExact(x.dim(.out), 2); + var gate = x.slice(.out, .{ .end = mid }); + var up = x.slice(.out, .{ .start = mid }); + + switch (activation) { + .swiglu_step => { + gate = gate.div(gate.negate().exp().addConstant(1)); + if (activation.swiglu_step.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + break :blk gate.mul(up); + }, + .swiglu => { + if (activation.swiglu.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + if (activation.swiglu.bias) |bias| up = up.addConstant(bias); + break :blk gate.div(gate.negate().exp().addConstant(1)).mul(up); + }, + .geglu => break :blk gelu(gate).mul(up), + .geglu_tanh => break :blk gate.gelu().mul(up), + else => unreachable, // already handled above + } + }, + }; +} + +/// More precise GELU approximation than the default Zig implementation, which is +/// based on tanh. This is the same approximation used in JAX. +pub fn gelu(x: Tensor) Tensor { + const xf = x.convert(.f32); + const cdf = xf.scale(-1.0 / @sqrt(@as(f64, 2))) + .erfc() + .scale(0.5); + return xf.mul(cdf).convert(x.dtype()); +} + +fn expandScales(scales: Tensor, shape: Shape) Tensor { + if (scales.count() == 1) return scales.asScalar().convert(.f32).broad(shape.withDtype(.f32)); + stdx.debug.assert(scales.rank() == shape.rank(), "reference scale rank must match weights", .{}); + var result = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = std.math.divCeil(i64, shape.dim(axis), result.dim(axis)) catch unreachable; + const expanded = result.shape().insert(axis + 1, .{factor}); + result = result.broadcast(expanded, Shape.range(expanded.rank(), .i64).remove(axis + 1).dims()) + .reshape(result.shape().setDim(axis, result.dim(axis) * factor)) + .slice(axis, .{ .end = shape.dim(axis) }); + } + return result.withTags(shape.tags()); +} + +pub fn dequantize(linear: zml.nn.Linear) Tensor { + var weight = linear.weight; + const q = linear.quantization orelse return weight.convert(.f32); + stdx.debug.assert(!q.swizzled_scales, "StableHLO reference requires plain block scales", .{}); + if (zml.nn.isPackedFp4(q.scheme, weight.dtype())) { + // StableHLO bitcast expands the low nibble first along the input axis. + weight = weight.bitCast(.f4e2m1).merge(.{ .unpacked = .{ linear.tag, .bitcast } }).renameTag(.unpacked, linear.tag); + } + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; + var result = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + if (q.global_scale) |scale| result = result.mul(scale.asMultiplier().broad(result.shape())); + return result; +} + +// E2M1 round-to-nearest, ties-to-even expressed independently of native +// float-to-FP4 lowering. Even encoding indices win midpoint ties. +fn roundFp4(x: Tensor) Tensor { + const magnitude = x.abs(); + var rounded = Tensor.scalar(6, .f32).broad(x.shape()); + const thresholds = [_]f32{ 5, 3.5, 2.5, 1.75, 1.25, 0.75, 0.25 }; + const values = [_]f32{ 4, 3, 2, 1.5, 1, 0.5, 0 }; + inline for (thresholds, values, 0..) |threshold, value, i| { + rounded = magnitude.cmp(if (i % 2 == 0) .LE else .LT, Tensor.scalar(threshold, .f32)) + .select(Tensor.scalar(value, .f32).broad(x.shape()), rounded); + } + return x.cmp(.LT, Tensor.scalar(0, .f32)).select(rounded.negate(), rounded); +} + +// Quantize and dequantize activation groups using tensor math, independently of +// Triton's quantizer. The return value is the represented value in FP32. +fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.DataType) Tensor { + if (opts.input_quantization == .none) return x.convert(dtype).convert(.f32); + const q = linear.quantization orelse return x.convert(dtype).convert(.f32); + const fixed_mxfp8 = opts.input_quantization == .mxfp8 or opts.input_quantization == .mxfp8_bf16; + const scheme = if (fixed_mxfp8) .fp8_block32 else q.scheme; + if (scheme == .nvfp4) { + const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); + stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); + // Keep both stored rounding boundaries when XLA fuses the quantizer. + const grouped = x.convert(.bf16).optimizationBarrier().convert(.f32).div(global.asScalar()) + .splitAxis(.in, .{ .group = @divExact(x.dim(.in), 16), .element = 16 }); + const scale = grouped.abs().max(.element).scale(1.0 / 6.0) + .minimum(Tensor.scalar(448, .f32)) + .convert(.f8e4m3fn).optimizationBarrier().convert(.f32).broad(grouped.shape()); + const divisor = scale.cmp(.NE, Tensor.scalar(0, .f32)).select(scale, Tensor.scalar(1, .f32).broad(scale.shape())); + return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) + .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); + } + if (opts.input_quantization == .automatic and (!opts.quantize_input or scheme == .mxfp4)) return x.convert(dtype).convert(.f32); + const output_dtype = if (fixed_mxfp8) .f8e4m3fn else linear.weight.dtype(); + const group: i64 = switch (scheme) { + .mxfp8, .fp8_block32 => 32, + .fp8_block128 => 128, + .fp8_per_channel, .fp8_per_tensor => x.dim(.in), + else => unreachable, + }; + const source = if (opts.input_quantization == .mxfp8_bf16) x.convert(.bf16) else x; + const grouped = source.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); + const epsilon: f32 = if (scheme == .fp8_block32) 1e-4 else 1e-10; + const max_value: f32 = if (output_dtype == .f8e4m3fnuz) 224 else 448; + var scale = grouped.abs().max(.element).maximum(Tensor.scalar(epsilon, .f32)).scale(1 / max_value); + if (scheme == .mxfp8 or scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); + const broad = scale.broad(grouped.shape()); + return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) + .convert(output_dtype).optimizationBarrier().convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); +} + +fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { + const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); + const selected = weight.gather(.{ .expert = ids }, .{}); + var result = roundedInput(x, linear, opts, dtype).dot(selected, .in); + if (linear.bias) |bias| { + const selected_bias = if (bias.rank() == 2) + bias.withTags(.{ .expert, .out }).gather(.{ .expert = ids }, .{}) + else + bias.withTags(.{.out}); + result = result.add(selected_bias.convert(.f32).broad(result.shape())); + } + return result; +} + +pub const Options = struct { + activation: Activation, + quantize_input: bool = false, + /// Model fixed backend contracts independently of optional input quantization: + /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. + /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. + input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, + routing_weight_placement: RoutingWeightPlacement = .after_down, +}; + +/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. +pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); + stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); + stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); + stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); + const tokens = input.dim(.t); + const k = topk_ids.dim(.topk); + // Nonlocal routes use expert zero for safe gathers; discard their full + // contribution after the down projection, including any projection bias. + const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); + const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); + const ids = safe_ids.reshape(.{ .route = tokens * k }); + const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); + const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) + .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) + .merge(.{ .route = .{ .token, .topk } }); + // Keep the projection's rounding boundary when XLA fuses activation/routing + // arithmetic; otherwise the reference can disagree with explicit BF16 GEMMs. + const projected = project(hidden, gate_up, ids, opts, input.dtype()).convert(input.dtype()).optimizationBarrier().convert(.f32); + var activated = activate(projected, opts.activation); + if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); + var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); + if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); + return output.convert(input.dtype()).convert(.f32) + .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) + .sum(.topk).squeeze(.topk).convert(input.dtype()); +} + +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { + stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); + return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .routing_weight_placement = opts.routing_weight_placement, + }); +} + +test "MoE compliance: FP4 midpoint rounding" { + const Local = struct { + fn call(x: Tensor) Tensor { + return roundFp4(x); + } + }; + const values = [_]f32{ -7, -5, -3.5, -2.5, -1.75, -1.25, -0.75, -0.25, 0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5, 7 }; + const expected = [_]f32{ -6, -4, -4, -2, -2, -1, -1, 0, 0, 0, 1, 1, 2, 2, 4, 4, 6 }; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const x: Tensor = .init(.{values.len}, .f32); + var exe = try platform.compileFn(allocator, io, Local.call, .{x}, .{}); + defer exe.deinit(); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); +} diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel new file mode 100644 index 00000000..44b56e08 --- /dev/null +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "moe.zig", + "tcl.zig", + ], +) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig new file mode 100644 index 00000000..f05727ce --- /dev/null +++ b/zml/kernels/moe/tcl/tcl.zig @@ -0,0 +1,250 @@ +//! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); + +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { + const topk_ids = routing.localIds(); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); + stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + + const gate_up = gate_up_linear.weight; + const down = down_linear.weight; + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + + const weights = routing.weights.convert(.f32); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; +} + +const Scales = struct { gate_up: Tensor, down: Tensor }; + +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + +fn fp8Scales(linear: zml.nn.Linear) Tensor { + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); + const w = linear.weight; + const scale = q.scales; + stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); + return scale; +} + +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; + +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); + } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} + +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); +} + +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; +} + +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; +} + +test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. + for ([_]i64{ 4, 16 }) |t| { + const k = 2; + const e = 8; + const d = 256; + const i = 64; + const x: Tensor = .init(.{ .t = t, .d = d }, .bf16); + const ids: Tensor = .init(.{ .t = t, .topk = k }, .i32); + const w: Tensor = .init(.{ .t = t, .topk = k }, .f32); + const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); + const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); + const Mod = struct { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { + return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ + .activation = act, + .quantize_input = false, + .routing_weight_placement = .after_down, + }); + } + }; + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); + defer exe.deinit(); + + var prng: std.Random.DefaultPrng = .init(@intCast(t)); + const r = prng.random(); + const hx = try allocator.alloc(bf16, @intCast(t * d)); + defer allocator.free(hx); + const hgu = try allocator.alloc(bf16, e * 2 * i * d); + defer allocator.free(hgu); + const hdn = try allocator.alloc(bf16, e * d * i); + defer allocator.free(hdn); + for (hx) |*v| v.* = .fromF32(r.floatNorm(f32) * 0.5); + for (hgu) |*v| v.* = .fromF32(r.floatNorm(f32) / 16); + for (hdn) |*v| v.* = .fromF32(r.floatNorm(f32) / 8); + const hids = try allocator.alloc(i32, @intCast(t * k)); + defer allocator.free(hids); + const hw = try allocator.alloc(f32, @intCast(t * k)); + defer allocator.free(hw); + for (0..@intCast(t)) |tok| for (0..k) |j| { + hids[tok * k + j] = @intCast((tok * 3 + j * 5) % e); + hw[tok * k + j] = if (j == 0) 0.7 else 0.3; + }; + + var bx: zml.Buffer = try .fromBytes(io, platform, x.shape(), std.mem.sliceAsBytes(hx)); + defer bx.deinit(); + var bids: zml.Buffer = try .fromBytes(io, platform, ids.shape(), std.mem.sliceAsBytes(hids)); + defer bids.deinit(); + var bw: zml.Buffer = try .fromBytes(io, platform, w.shape(), std.mem.sliceAsBytes(hw)); + defer bw.deinit(); + var bgu: zml.Buffer = try .fromBytes(io, platform, gu.shape(), std.mem.sliceAsBytes(hgu)); + defer bgu.deinit(); + var bdn: zml.Buffer = try .fromBytes(io, platform, dn.shape(), std.mem.sliceAsBytes(hdn)); + defer bdn.deinit(); + var result = try exe.eval(allocator, io, .{ bx, bids, bw, bgu, bdn }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const out = host.items(bf16); + + for (0..@intCast(t)) |tok| { + var expected: [d]f32 = @splat(0); + for (0..k) |j| { + const ex: usize = @intCast(hids[tok * k + j]); + var act: [i]f32 = undefined; + for (0..i) |n| { + var g: f32 = 0; + var u: f32 = 0; + for (0..d) |m| { + const xv = hx[tok * d + m].toF32(); + g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); + u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); + } + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; + } + for (0..d) |m| { + var y: f32 = 0; + for (0..i) |n| y += act[n] * hdn[(ex * d + m) * i + n].toF32(); + expected[m] += hw[tok * k + j] * y; + } + } + for (expected, out[tok * d ..][0..d]) |want, got| try std.testing.expectApproxEqAbs(want, got.toF32(), 2e-2 + 2e-2 * @abs(want)); + } + } +} diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig new file mode 100644 index 00000000..c74406c3 --- /dev/null +++ b/zml/kernels/moe/tests.zig @@ -0,0 +1,752 @@ +//! Shared public-entry-point tests. Hardware absence is the only runtime skip: +//! supported cases are explicit, and errors in those cases must fail the test. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; +const reference = @import("stablehlo/stablehlo.zig"); + +const Case = struct { + backend: zml.moe.Backend, + activation: Activation = .{ .swiglu = .{} }, + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, + quantize_input: bool = false, + placement: Placement = .after_down, + bias: bool = false, + non_unit_scales: bool = false, + tokens: i64 = 1, + topk: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, + expert_mesh: *const zml.Mesh, + + fn inputShape(c: Case) Shape { + const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + return shape.withPartitioning(c.expert_mesh, spec); + } + + pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { + std.debug.assert(c.backend.isAvailable(platform)); + const x: Tensor = .fromShape(c.inputShape()); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); + } + + pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { + const x: Tensor = .fromShape(c.inputShape()); + + var host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + switch (c.activation_dtype) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType()), 0..) |*value, i| { + const f = @as(f32, @floatFromInt(@as(i32, @intCast((i * 13 + i / 128) % 23)) - 11)) / 32; + value.* = if (dtype == .bf16) .fromF32(f) else @floatCast(f); + } + }, + else => unreachable, + } + + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var output: zml.Bufferized(Outputs) = undefined; + runner.run(io, .{input}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Outputs, &output); + try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); + + // BF16 intermediate rounding and different GEMM reduction orders. Every + // element must pass; near-zero values use the absolute bound. + try zml.testing.expectClose(io, output.expected, output.actual, .{ + // Share the NVFP4 comparison budget across CUDA backends. + .absolute_tolerance = if (c.scheme == .nvfp4) 0.078125 else 0.015625, + .relative_tolerance = 0.02, + .minimum_close_fraction = 1, + }); + } +}; + +/// Generate a reproducible Tensor with a simple pattern. +/// +/// Generate as many elements as the shape requires, with this formula: +/// value[i] = (((i * multiplier + 3) % 17) - 8) / divisor +/// +/// Then reshape to the requested shape and convert to f32. +fn generateDeterministicTensor(shape: Shape, multiplier: i64, divisor: f64) Tensor { + return Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32) + .mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).addConstant(-8) + .convert(.f32).scale(1 / divisor).reshape(shape.withDtype(.f32)); +} + +/// Select exactly representable FP4 values with distinct even/odd sequences. +fn generateFp4Tensor(shape: Shape, multiplier: i64) Tensor { + const values = Tensor.constantTensor(.init(.{ .value = 16 }, .f32), std.mem.sliceAsBytes(&[_]f32{ + 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6, + })); + const index = Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32); + const pair = index.div(Tensor.scalar(2, .i32)); + // Prime periods distinguish experts, output rows, and gate/up halves. + const even = pair.mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).remainder(Tensor.scalar(16, .i32)); + const odd = pair.mul(Tensor.scalar(3, .i32)).addConstant(7) + .remainder(Tensor.scalar(19, .i32)).remainder(Tensor.scalar(16, .i32)); + const indices = index.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)).select(even, odd); + return values.gather(.{ .value = indices }, .{}).convert(.f4e2m1).reshape(shape.withDtype(.f4e2m1)); +} + +/// Create a Linear fixture with a reproducible pattern, optionally quantized and biased. +fn linear(shape: Shape, scheme: ?zml.Quantization.Scheme, bias: bool, multiplier: i64) zml.nn.Linear { + const fixture_bias = if (bias) generateDeterministicTensor(shape.remove(2), 3, 32).convert(shape.dtype()) else null; + + const s = scheme orelse return .{ + .weight = generateDeterministicTensor(shape, multiplier, 16).convert(shape.dtype()), + .tag = shape.tag(2), + .bias = fixture_bias, + }; + + const weight = switch (s) { + .mxfp4, .nvfp4 => generateFp4Tensor(shape, multiplier), + else => generateDeterministicTensor(shape, multiplier, 16).convert(.f8e4m3fn), + }; + + const scale_shape: Shape = switch (s) { + .mxfp4, .mxfp8 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 32) }, .f8e8m0), + .nvfp4 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 16) }, .f8e4m3fn), + .fp8_per_tensor => .init(.{ shape.dim(0), 1, 1 }, .f32), + .fp8_per_channel => .init(.{ shape.dim(0), shape.dim(1), 1 }, .f32), + .fp8_block128 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 128), @divExact(shape.dim(2), 128) }, .f32), + .fp8_block32 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 32), @divExact(shape.dim(2), 32) }, .f32), + }; + + const scales = Tensor.arange(.{ .end = @intCast(scale_shape.count()) }, .i32) + .remainder(Tensor.scalar(2, .i32)).convert(.f32).addConstant(1).scale(1.0 / 16.0) + .reshape(scale_shape.withDtype(.f32)).convert(scale_shape.dtype()); + + const input_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32), .operation = .multiply } else null; + const global_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32).broad(.init(.{ .expert = shape.dim(0) }, .f32)), .operation = .multiply } else null; + + return .{ + .weight = weight, + .tag = shape.tag(2), + .bias = fixture_bias, + .quantization = .{ + .scheme = s, + .scales = scales, + .input_scale = input_scale, + .global_scale = global_scale, + }, + }; +} + +const ExpertFixture = struct { + gate_up: zml.nn.Linear, + down: zml.nn.Linear, +}; + +/// Canonical weights use logical dimensions, concatenated gate/up, and plain scales. +fn makeExpertFixture(c: Case) ExpertFixture { + const projection_width: i64 = switch (c.activation) { + .silu, .gelu, .relu => c.intermediate, + else => 2 * c.intermediate, + }; + var fixture: ExpertFixture = .{ + .gate_up = linear(.init(.{ .expert = c.experts, .dout = projection_width, .d = c.width }, c.activation_dtype), c.scheme, c.bias, 7), + .down = linear(.init(.{ .expert = c.experts, .d = c.width, .dout = c.intermediate }, c.activation_dtype), c.scheme, c.bias, 11), + }; + if (c.non_unit_scales) { + for ([_]*zml.nn.Linear{ &fixture.gate_up, &fixture.down }, 0..) |projection, i| { + const q = &projection.quantization.?; + std.debug.assert(q.scheme == .nvfp4); + // Exercise both scale polarity and distinct per-expert multipliers. + q.input_scale = .{ .value = Tensor.scalar(@as(f32, if (i == 0) 2 else 0.5), .f32), .operation = .divide }; + q.global_scale = .{ + .value = Tensor.arange(.{ .end = @intCast(c.experts) }, .i32) + .remainder(Tensor.scalar(2, .i32)).addConstant(1).convert(.f32).withTags(.{.expert}), + .operation = .divide, + }; + } + } + return fixture; +} + +fn interleave(tensor: Tensor, rows: i64) Tensor { + return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) + .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); +} + +/// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> +/// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. +fn swizzleScales(scales: Tensor) Tensor { + return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); +} + +fn packFp4Bytes(weight: Tensor) Tensor { + std.debug.assert(weight.dtype() == .f4e2m1); + return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); +} + +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { + return switch (layout.gate_up) { + .concatenated => tensor, + .row_interleaved_1 => interleave(tensor, 1), + .row_interleaved_8 => interleave(tensor, 8), + }; +} + +/// Adapt a copy of both projections to the backend's storage and layout contract. +fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertFixture { + var prepared = canonical; + const scheme = canonical.gate_up.quantizationScheme(); + const layout = backend.expertsLayout(scheme) catch |err| zml.stdx.debug.panic("fixture layout: {}", .{err}); + // Plain activations have one projection, so gate/up ordering does not apply. + const gated = canonical.gate_up.weight.dim(1) == 2 * canonical.down.weight.dim(2); + if (gated) { + prepared.gate_up.weight = prepareGateUp(canonical.gate_up.weight, layout); + if (canonical.gate_up.bias) |bias| prepared.gate_up.bias = prepareGateUp(bias.appendAxes(.{.dummy}), layout).squeeze(.dummy); + if (prepared.gate_up.quantization) |*q| q.scales = prepareGateUp(q.scales, layout); + } + + for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { + switch (layout.packing) { + .swizzled_scales, .flashinfer_nvfp4 => { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + }, + .mxfp4_interleaved8 => { + const q = &projection.quantization.?; + const sf = q.scales; + q.scales = sf.bitCast(.u8) + .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) + .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) + .withTags(sf.shape().transpose(.{ 0, 2, 1 })); + }, + .plain => {}, + } + switch (backend) { + .cute => { + projection.weight = packFp4Bytes(projection.weight); + const q = &projection.quantization.?; + if (scheme == .nvfp4) { + q.scales = q.scales.bitCast(.u8); + // CuTe takes the input quantization multiplier and a GEMM + // output multiplier that folds in its inverse. + const input_scale = q.input_scale.?.asMultiplier(); + q.global_scale = .{ .value = q.global_scale.?.asMultiplier().mul(input_scale), .operation = .multiply }; + q.input_scale = .{ .value = Tensor.scalar(1, .f32).div(input_scale), .operation = .multiply }; + } + }, + .triton => if (scheme == .mxfp4) { + projection.weight = packFp4Bytes(projection.weight); + }, + .mosaic_tpu => { + projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); + }, + .metal => if (projection.quantization) |*q| { + if (q.scheme != .nvfp4) q.scales = q.scales.convert(.bf16); + }, + else => {}, + } + } + + if (backend == .metal and scheme == .nvfp4) { + const global = &prepared.gate_up.quantization.?.global_scale.?; + global.value = global.value.appendAxes(.{.proj}).repeat1d(.proj, 2); + } + return prepared; +} + +const Outputs = struct { actual: Tensor, expected: Tensor }; + +/// Forward a fixture through the MoE implementation under test, and also through the StableHLO +/// reference implementation. Return both outputs for comparison. +fn forward(input: Tensor, c: Case) Outputs { + const canonical = makeExpertFixture(c); + const prepared = prepareForBackend(canonical, c.backend); + + const route_shape = Shape.init(.{ .t = c.tokens, .topk = c.topk }, .i32); + const route = Tensor.arange(.{ .end = @intCast(route_shape.count()) }, .i32).reshape(route_shape); + + // Only six of eight experts receive routes, with nonuniform counts. + const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); + + const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); + const options: GlobalOptions = .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + }; + + const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ + .in_specs = .{ + .input = data_spec, + .ids = data_spec, + .weights = data_spec, + .gate_up = expert_spec, + .down = expert_spec, + }, + .out_specs = data_spec, + }, options); + // Keep the oracle input replicated so token sharding does not change its GEMM rounding. + const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + .input_quantization = switch (c.backend) { + .metal => .none, + .cute => if (c.scheme == .nvfp4) .automatic else .mxfp8_bf16, + .triton => if (c.scheme == .mxfp4 and zml.Compiler.current().platform.target == .cuda) .mxfp8 else .automatic, + else => .automatic, + }, + }); + + return .{ .actual = actual, .expected = expected }; +} + +const gated_activations = [_]Activation{ + .{ .swiglu = .{} }, + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .bias = 1 } }, + .{ .swiglu = .{ .limit = 0.25, .bias = 1 } }, + .{ .swiglu_step = .{} }, + .{ .swiglu_step = .{ .limit = 0.25 } }, + .geglu, + .geglu_tanh, +}; +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .limit = 7 } }, +}; + +const FixtureShape = struct { + tokens: i64 = 1, + topk: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, +}; + +// Every option combination runs with 1, 17, and 34 flattened tokens. +// Shapes and floating-point parameter values are representative finite sets. +const fixture_shapes = [_]FixtureShape{ + .{}, + .{ .tokens = 17, .topk = 2 }, + .{ .tokens = 34, .topk = 3 }, +}; +const specialized_shapes = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 34, .topk = 2 }, +}; +// Exercise direct decode routing with multiple experts; a single route is unsupported. +const specialized_shapes_cute = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 1, .topk = 2 }, +} ++ specialized_shapes[1..].*; + +const Format = struct { + non_unit_scales: bool = false, + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, +}; + +const Matrix = struct { + formats: []const Format = &.{.{}}, + activations: []const Activation, + quantize_inputs: []const bool = &.{false}, + placements: []const Placement = &.{.after_down}, + biases: []const bool = &.{false}, + shapes: []const FixtureShape = &fixture_shapes, + + pub fn count(self: Matrix) usize { + return self.formats.len * self.activations.len * self.quantize_inputs.len * + self.placements.len * self.biases.len * self.shapes.len; + } + + /// Shapes vary fastest, matching the nested loops in the backend tests. + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { + std.debug.assert(index < self.count()); + var remaining = index; + const shape = self.shapes[remaining % self.shapes.len]; + remaining /= self.shapes.len; + const bias = self.biases[remaining % self.biases.len]; + remaining /= self.biases.len; + const placement = self.placements[remaining % self.placements.len]; + remaining /= self.placements.len; + const quantize_input = self.quantize_inputs[remaining % self.quantize_inputs.len]; + remaining /= self.quantize_inputs.len; + const activation = self.activations[remaining % self.activations.len]; + remaining /= self.activations.len; + const format = self.formats[remaining]; + return .{ + .backend = backend, + .activation = activation, + .scheme = format.scheme, + .activation_dtype = format.activation_dtype, + .non_unit_scales = format.non_unit_scales, + .quantize_input = quantize_input, + .placement = placement, + .bias = bias, + .tokens = shape.tokens, + .topk = shape.topk, + .width = shape.width, + .intermediate = shape.intermediate, + .experts = shape.experts, + .expert_mesh = expert_mesh, + }; + } +}; + +const Options = struct { + producers: usize = 16, + consumers: usize = 1, + case_queue_capacity: usize = 16, + executable_queue_capacity: usize = 16, +}; + +const CaseJob = struct { + index: usize, + case: Case, +}; + +const ExecutableJob = struct { + job: CaseJob, + exe: zml.Exe, +}; + +const Result = struct { + case: Case, + outcome: union(enum) { + passed, + skipped, + compilation_failed: anyerror, + check_failed: anyerror, + }, +}; + +const Pipeline = struct { + platform: *const zml.Platform, + + fn generateCases(self: *const Pipeline, io: std.Io, backend: zml.moe.Backend, matrix: Matrix, case_queue: *std.Io.Queue(CaseJob), results: []?Result) std.Io.Cancelable!void { + defer case_queue.close(io); + const available = backend.isAvailable(self.platform); + + var index: usize = 0; + for (0..matrix.count()) |local_index| { + const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); + if (available) { + case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + } else { + results[index] = .{ .case = c, .outcome = .skipped }; + } + index += 1; + } + } + + fn compileExecutablesWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, case_queue: *std.Io.Queue(CaseJob), executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + const job = case_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + var exe = job.case.compile(allocator, io, self.platform) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .compilation_failed = err } }; + continue; + }; + // Ownership transfers only on a successful enqueue. + executable_queue.putOne(io, .{ .job = job, .exe = exe }) catch |err| { + exe.deinit(); + return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + }; + } + } + + fn runTestCaseWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + var compiled = executable_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + defer compiled.exe.deinit(); + const job = compiled.job; + job.case.check(allocator, io, self.platform, &compiled.exe) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .check_failed = err } }; + continue; + }; + results[job.index] = .{ .case = job.case, .outcome = .passed }; + } + } + + fn run(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, backend: zml.moe.Backend, matrix: Matrix, options: Options) !void { + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const cases_buffer = try arena.allocator().alloc(CaseJob, options.case_queue_capacity); + const executables_buffer = try arena.allocator().alloc(ExecutableJob, options.executable_queue_capacity); + const results = try arena.allocator().alloc(?Result, matrix.count()); + @memset(results, null); + + var case_queue: std.Io.Queue(CaseJob) = .init(cases_buffer); + var executable_queue: std.Io.Queue(ExecutableJob) = .init(executables_buffer); + + var enumerator: std.Io.Group = .init; + var producers: std.Io.Group = .init; + var consumers: std.Io.Group = .init; + defer { + case_queue.close(io); + executable_queue.close(io); + enumerator.cancel(io); + producers.cancel(io); + consumers.cancel(io); + // On partial startup or cancellation, queued executables still own resources. + while (executable_queue.getOneUncancelable(io)) |compiled| { + compiled.exe.deinit(); + } else |_| {} + } + + for (0..options.consumers) |_| try consumers.concurrent(io, runTestCaseWorker, .{ self, io, allocator, &executable_queue, results }); + for (0..options.producers) |_| try producers.concurrent(io, compileExecutablesWorker, .{ self, io, allocator, &case_queue, &executable_queue, results }); + try enumerator.concurrent(io, generateCases, .{ self, io, backend, matrix, &case_queue, results }); + + try enumerator.await(io); + try producers.await(io); + // Closed queues drain before getOne returns error.Closed. + executable_queue.close(io); + try consumers.await(io); + + try report(results); + } + + fn report(results: []const ?Result) !void { + var failed: usize = 0; + for (results, 0..) |result, index| { + const r = result orelse return error.MissingCaseResult; + switch (r.outcome) { + .passed, .skipped => {}, + inline .compilation_failed, .check_failed => |err, stage| { + failed += 1; + std.debug.print("MoE case {}/{} {s}: {any}\nError: {}\n", .{ + index + 1, results.len, @tagName(stage), r.case, err, + }); + }, + } + } + if (failed != 0) return error.MoeComplianceFailed; + } +}; + +test "Triton compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{ + .{}, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + // BF16 ignores this option; exercise both accepted settings. + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); + + if (platform.target != .cuda) { + // Other platforms retain the general MXFP4 path and its full contract. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &all_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); + return; + } + + // Preserve the 12 supported cases from the former general MXFP4 matrix. + // Its other 252 cases require activations or projection bias unsupported + // by the simplified path. The former specialized matrix remains below. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{gated_activations[1]}, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +// Keep triton mxfp4 separate because it only supports a subset for now +test "Triton MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const pipeline: Pipeline = .{ .platform = platform }; + + // The merged backend uses the former specialized MXFP4 path: clipped + // SwiGLU without projection bias, with unconditional MXFP8 activations. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +test "Triton MoE prefill launch buckets fit shared memory" { + const platform = zml.testing.env(); + if (!zml.moe.Backend.triton.isAvailable(platform) or platform.target != .rocm) return error.SkipZigTest; + + for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { + const case: Case = .{ + .backend = .triton, + .expert_mesh = platform.replicated_mesh, + .tokens = tokens, + .width = 256, + .intermediate = 128, + .topk = 2, + }; + var exe = try case.compile(std.testing.allocator, std.testing.io, platform); + defer exe.deinit(); + try case.check(std.testing.allocator, std.testing.io, platform, &exe); + } +} + +test "FlashInfer compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Only default SwiGLU, no linear bias, and routing after the down projection. + try pipeline.run(std.testing.io, std.testing.allocator, .flashinfer_cutlass, .{ + .formats = &.{.{ .scheme = .nvfp4 }}, + .activations = &.{.{ .swiglu = .{} }}, + }, .{}); +} + +test "specialized CuTe MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes_cute, + }, .{}); +} + +test "CuTe NVFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (!zml.moe.cute.isNvfp4Supported(platform)) return error.SkipZigTest; + + const pipeline: Pipeline = .{ .platform = platform }; + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ + .formats = &.{ + .{ .scheme = .nvfp4 }, + .{ .scheme = .nvfp4, .non_unit_scales = true }, + }, + .activations = &.{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } } }, + .placements = &.{ .before_down, .after_down }, + .shapes = &.{ + .{ .width = 5120, .intermediate = 2304, .tokens = 1, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 34, .topk = 3 }, + }, + }, .{}); +} + +test "Fly compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // These constraints keep execution on Fly rather than its Triton fallback. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &gated_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes, + }, .{}); +} + +test "Fly general projection shapes" { + const pipeline: Pipeline = .{ .platform = zml.testing.env() }; + + // Cover GEMV, sorted MFMA with 16- and 32-row tiles, and wider projections. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{.{ .swiglu = .{} }}, + .placements = &.{.before_down}, + .shapes = &.{ + .{ .width = 384, .intermediate = 128, .experts = 2 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 7 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 1024 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2, .tokens = 7 }, + }, + }, .{}); +} + +test "Metal compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Quantized Metal kernels require BF16 activations and do not support bias. + try pipeline.run(std.testing.io, std.testing.allocator, .metal, .{ + .formats = &.{ + .{}, + .{ .activation_dtype = .f16 }, + .{ .activation_dtype = .f32 }, + .{ .scheme = .nvfp4 }, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + }, .{}); +} + +test "Mosaic TPU compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // canonicalizeDown currently requires a gated projection; quantized weights + // are not supported by gmmDType. Plain activations need a backend fix first. + try pipeline.run(std.testing.io, std.testing.allocator, .mosaic_tpu, .{ + .formats = &.{ .{}, .{ .activation_dtype = .f16 }, .{ .activation_dtype = .f32 } }, + .activations = &gated_activations, + .biases = &.{ false, true }, + }, .{}); +} diff --git a/zml/kernels/moe/triton/BUILD.bazel b/zml/kernels/moe/triton/BUILD.bazel new file mode 100644 index 00000000..8dda7f88 --- /dev/null +++ b/zml/kernels/moe/triton/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "moe.zig", + "triton.zig", + ], +) diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig new file mode 100644 index 00000000..6e4006fa --- /dev/null +++ b/zml/kernels/moe/triton/moe.zig @@ -0,0 +1,1178 @@ +//! Triton MoE kernel declarations — direct ports of the `@triton.jit` +//! functions in `triton_kernels/moe.py`. Each kernel here is a +//! `tri.Kernel(...)` declaration with its own inline config struct (no +//! defaults — every field is required at call time). Production callers +//! invoke these via `K.call(inputs, outputs, opts)`; offline tooling reaches the TTIR +//! string via `K.emit(allocator, ctx, cfg)`. + +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; + +const log = std.log.scoped(.@"zml/moe/triton_kernels"); + +/// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards +/// that keep dynamic strides aligned for tt.load/store. +fn blockFloor16(v: Value) Value { + return v.div(16).mul(16); +} + +// ============================================================================= +// per_token_group_quant_fp8 +// ============================================================================= + +/// Direct port of Python `per_token_group_quant_fp8` — per-token-group +/// float8 quantization. +pub const PerTokenGroupQuantFp8 = struct { + pub const Cfg = struct { + input_dtype: DType, + output_dtype: DType, + scale_dtype: DType, + block: usize, + quant_scheme: zml.Quantization.Scheme, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "per_token_group_quant_fp8", + .inputs = &.{ "y_ptr", "group_size_ptr", "y_num_columns_ptr", "y_row_stride_ptr", "eps_ptr" }, + .outputs = &.{ "y_q", "y_s" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .y_ptr = .{ .ptr = cfg.input_dtype }, + .group_size_ptr = .{ .ptr = .i64 }, + .y_num_columns_ptr = .{ .ptr = .i64 }, + .y_row_stride_ptr = .{ .ptr = .i64 }, + .eps_ptr = .{ .ptr = .f32 }, + .y_q_ptr = .{ .ptr = cfg.output_dtype }, + .y_s_ptr = .{ .ptr = cfg.scale_dtype }, + }); + + const block: i64 = @intCast(cfg.block); + const out_dt = cfg.output_dtype; + const fp8_max: f32 = switch (out_dt) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => return error.InvalidMlir, + }; + const scale_dt = cfg.scale_dtype; + + const group_size = b.load(a.group_size_ptr); + const y_num_columns = b.load(a.y_num_columns_ptr); + const y_row_stride = b.load(a.y_row_stride_ptr); + const eps = b.load(a.eps_ptr); + + const groups_per_row = y_num_columns.div(group_size); + // g_id is i32 from `tl.program_id`; the i64 promotion only happens + // inside the `(row|row_g_id).to(tl.int64)` casts in the Python source. + // y_s_ptr_shifted reuses the raw i32 `g_id` directly (no extsi). + const pid_i32 = b.programId(.x); + const g_id = pid_i32.to(.i64); + + // y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size + // Python emit order: divsi, remsi, muli (row * stride), muli (row_g_id * group_size), addi. + const row = g_id.div(groups_per_row); + const row_g_id = g_id.rem(groups_per_row); + const row_off = row.mul(y_row_stride); + const grp_off = row_g_id.mul(group_size); + const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); + const y_q_ptr_shifted = a.y_q_ptr.addPtr(g_id.mul(group_size)); + // y_s_ptr += g_id (raw i32, no cast — matches `y_s_ptr += g_id` in Python). + const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); + + // cols = arange(0, BLOCK) — i64 for both pointer offsets and mask + // comparison against the i64 `group_size` (matches Python's + // `cols < group_size` after Triton's auto-promotion). + const cols_i32 = b.arange(0, block, .i32); + const cols_i64 = cols_i32.to(.i64); + const mask = cols_i64.lt(group_size); + + // Python: `tl.load(y_ptr + cols, mask=mask, other=0.0)` — explicit + // 3-operand form. Use `.other` to materialize the const-zero tensor + // (without it, `tt.load` is 2-operand and Python's emit doesn't match). + const y_load_ptrs = y_ptr_shifted.addPtr(cols_i64); + const y_other = b.zeros(&.{block}, cfg.input_dtype); + const y = b.loadOpts(y_load_ptrs, .{ .mask = mask, .other = y_other }).to(.f32); + + // _absmax = max(tl.max(tl.abs(y)), eps) — `b.max(...).maximum(eps)` + // resolves to `arith.maxnumf` (matches Python's `tl.maximum`). + const absmax = b.max(b.absf(y)).maximum(eps); + + // scale_raw = _absmax * (1.0 / fp8_max) + const scale_raw = absmax.mul(1.0 / fp8_max); + const y_s: Value, const stored_scale: Value = switch (cfg.quant_scheme) { + .mxfp8 => blk: { + const exponent = b.ceil(b.log2(scale_raw)); + // E8M0 stores the biased exponent as a byte. + break :blk .{ b.exp2(exponent), exponent.add(127).to(.i32).to(.i8) }; + }, + .fp8_per_channel, .fp8_per_tensor, .fp8_block128 => .{ scale_raw, scale_raw.to(scale_dt) }, + .fp8_block32 => blk: { + // Match quantizeBlockFp8(..., .f8e8m0), but store the scales as FP32 here. + // It is unclear whether power-of-two scaling is best for this kernel, + // but it keeps activation quantization consistent with that function. + // Round up from the FP32 bits, including values just above a power of two. + const bits = scale_raw.bitCast(.i32); + const exponent = bits.div(0x800000); + const fractional = bits.bitAnd(0x7fffff).ne(0).to(.i32); + const scale = exponent.add(fractional).mul(0x800000).bitCast(.f32); + break :blk .{ scale, scale.to(scale_dt) }; + }, + .mxfp4, .nvfp4 => unreachable, + }; + + // y_q = clamp(y / y_s, fp8_min, fp8_max).to(output_dtype) + const y_div = y.div(y_s); + const clamped = b.clampf( + y_div, + b.splat(-fp8_max, &.{block}), + b.splat(fp8_max, &.{block}), + ); + const y_q = clamped.to(out_dt); + + const y_q_ptrs = y_q_ptr_shifted.addPtr(cols_i64); + b.storeOpts(y_q_ptrs, y_q, .{ .mask = mask }); + b.store(y_s_ptr_shifted, stored_scale); + } +}; + +// ============================================================================= +// write_zeros_to_output — helper called by FusedMoe +// ============================================================================= + +/// Direct port of Python `write_zeros_to_output`. Stores a masked zero-tile +/// for blocks whose expert is `-1` (no expert assigned). `stride_cn` is +/// hardcoded to 1 (contiguous last dim), so the N-axis offset is just +/// `offs_cn` itself. +fn writeZerosToOutput( + b: *Builder, + c_ptr: Value, + stride_cm: Value, + pid_n: Value, + n: Value, + offs_token: Value, + token_mask: Value, + block_size_m: i64, + block_size_n: i64, + compute_type: DType, +) void { + const accumulator = b.zeros(&.{ block_size_m, block_size_n }, compute_type); + const offs_cn = pid_n.mul(block_size_n).add(b.arange(0, block_size_n, .i64)); + + // c_ptrs in two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right evaluation. `stride_cm * offs_token[:, None]` puts the + // splatted stride on the LHS of arith.muli (Python source order). + const offs_token_col = b.expandDims(offs_token, 1); + const cm_col = stride_cm.mul(offs_token_col); + const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const cn_off_2d = b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(cn_off_2d); + + // c_mask: Python emit order — expand_dims, splat, cmpi, broadcast, broadcast, andi. + const token_mask_col = b.expandDims(token_mask, 1); + const n_splat = b.splat(n, &.{ 1, block_size_n }); + const offs_cn_lt_2d = offs_cn_row.lt(n_splat); + const token_mask_full = b.broadcastTo(token_mask_col, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full.bitAnd(offs_cn_lt_full) }); +} + +// ============================================================================= +// fused_moe_kernel +// ============================================================================= + +/// Routed GEMM supporting BF16, MXFP4, MXFP8, and per-tensor, per-channel, or +/// 128x128 block-scaled FP8 weights. +pub const FusedMoe = struct { + pub const Cfg = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType, + a_scale_dtype: ?DType, + b_scale_dtype: ?DType, + b_bias_dtype: ?DType, + routing_weights_dtype: ?DType, + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + top_k: usize, + naive_block_assignment: bool, + compute_type: DType, + quant_scheme: ?zml.Quantization.Scheme, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "fused_moe_kernel", + .inputs = &.{ + "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", + "routing_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", + "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", + "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", + "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", + }, + .outputs = &.{"c"}, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const fp8_block_size: usize = if (cfg.quant_scheme == .fp8_block32) 32 else 128; + if ((cfg.quant_scheme == .fp8_block128 or cfg.quant_scheme == .fp8_block32) and cfg.block_size_k != fp8_block_size) { + log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K={}", .{fp8_block_size}); + return error.InvalidMlir; + } + + if ((cfg.quant_scheme == .mxfp4 or cfg.quant_scheme == .mxfp8) and cfg.block_size_k % 32 != 0) { + log.err("fused_moe_kernel: MXFP4/MXFP8 require BLOCK_SIZE_K divisible by 32", .{}); + return error.InvalidMlir; + } + + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = cfg.a_dtype }, + .b_ptr = .{ .ptr = cfg.b_dtype }, + // The fixed signature requires concrete pointer types even for absent, + // unused inputs. FP32 matches the caller's placeholder tensors. + .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse .f32 }, + .a_scale_ptr = .{ .ptr = cfg.a_scale_dtype orelse .f32 }, + .b_scale_ptr = .{ .ptr = cfg.b_scale_dtype orelse .f32 }, + .routing_weights_ptr = .{ .ptr = cfg.routing_weights_dtype orelse .f32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_padded_ptr = .{ .ptr = .i32 }, + .N_ptr = .{ .ptr = .i64 }, + .K_ptr = .{ .ptr = .i64 }, + .EM_ptr = .{ .ptr = .i64 }, + .num_valid_tokens_ptr = .{ .ptr = .i64 }, + .stride_am_ptr = .{ .ptr = .i64 }, + .stride_be_ptr = .{ .ptr = .i64 }, + .stride_bn_ptr = .{ .ptr = .i64 }, + .stride_cm_ptr = .{ .ptr = .i64 }, + .stride_asm_ptr = .{ .ptr = .i64 }, + .stride_ask_ptr = .{ .ptr = .i64 }, + .stride_bse_ptr = .{ .ptr = .i64 }, + .stride_bsk_ptr = .{ .ptr = .i64 }, + .stride_bsn_ptr = .{ .ptr = .i64 }, + .stride_bbe_ptr = .{ .ptr = .i64 }, + .stride_bbn_ptr = .{ .ptr = .i64 }, + .c_ptr = .{ .ptr = cfg.c_dtype }, + }); + + const block_size_m: i64 = @intCast(cfg.block_size_m); + const block_size_n: i64 = @intCast(cfg.block_size_n); + const block_size_k: i64 = @intCast(cfg.block_size_k); + const weight_packing: i64 = if (cfg.quant_scheme == .mxfp4) 2 else 1; + const block_size_bk = @divExact(block_size_k, weight_packing); + const group_size_m: i64 = @intCast(cfg.group_size_m); + const top_k: i64 = @intCast(cfg.top_k); + const compute_type = cfg.compute_type; + + // Matrix dimensions, strides, and padded routing capacity are checked + // for 16-element alignment by the caller. + const n_raw = b.load(a.N_ptr); + const k_raw = b.load(a.K_ptr); + const em_raw = b.load(a.EM_ptr); + const num_valid_tokens = b.load(a.num_valid_tokens_ptr); + const stride_am_raw = b.load(a.stride_am_ptr); + const stride_be_raw = b.load(a.stride_be_ptr); + const stride_bn_raw = b.load(a.stride_bn_ptr); + const stride_cm_raw = b.load(a.stride_cm_ptr); + const n_block = blockFloor16(n_raw); + const k_block = blockFloor16(k_raw); + const em_block = blockFloor16(em_raw); + const stride_am_block = blockFloor16(stride_am_raw); + const stride_be_block = blockFloor16(stride_be_raw); + const stride_bn_block = blockFloor16(stride_bn_raw); + const stride_cm_block = blockFloor16(stride_cm_raw); + // stride_ak / stride_bk / stride_cn are all 1 in this port — the A/B/C + // inner dimension is contiguous, so those mul-by-1 terms are elided. + + // pid grouping. Python keeps `pid` as i32 and lets the divsi auto- + // promote at the use site — emit the cdivs first, then extsi at the + // first i64 use. Manual order keeps `extsi pid` after the cdivs in TTIR. + const pid_i32 = b.programId(.x); + const num_pid_m = em_block.cdiv(block_size_m); + const num_pid_n = n_block.cdiv(block_size_n); + const num_pid_in_group = num_pid_n.mul(group_size_m); + const pid = pid_i32.to(.i64); + const group_id = pid.div(num_pid_in_group); + const first_pid_m = group_id.mul(group_size_m); + const gsm_actual = num_pid_m.sub(first_pid_m).minimum(group_size_m); + const pid_mod_in_group = pid.rem(num_pid_in_group); + const pid_m = first_pid_m.add(pid_mod_in_group.rem(gsm_actual)); + const pid_n = pid_mod_in_group.div(gsm_actual); + + // Python source order: arange first, then load + early-return. + // Match emit order so the make_range op lands before the cf.cond_br. + const offs = b.arange(0, block_size_m, .i64); + // Order: load (i32), muli, extsi (Python emits the muli before the + // i32→i64 extsi at the cmpi). + const num_tokens_post_padded_i32 = b.load(a.num_tokens_post_padded_ptr); + const pid_m_block = pid_m.mul(block_size_m); + const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); + const out_of_range = pid_m_block.ge(num_tokens_post_padded); + + b.returnIfOpts(out_of_range, .{ .inline_return = true }); + + const offs_token = if (cfg.naive_block_assignment) + b.select( + offs.eq(0), + pid_m.splatTo(&.{block_size_m}), + num_valid_tokens.splatTo(&.{block_size_m}), + ) + else off: { + const ids_ptrs = a.sorted_token_ids_ptr + .addPtr(pid_m.mul(block_size_m).add(offs)); + break :off b.load(ids_ptrs).to(.i64); + }; + const token_mask = offs_token.lt(num_valid_tokens); + + // off_experts = expert_ids[pid_m].to(i64). pid_m is i64; Python's + // `expert_ids_ptr + pid_m` keeps the offset as i64. + const off_experts = b.load(a.expert_ids_ptr.addPtr(pid_m)).to(.i64); + const is_dead = off_experts.eq(-1); + + // Python: if off_experts == -1: write_zeros_to_output(...); return + var dead_ret = b.openReturnIfOpts(is_dead, .{ .inline_return = true }); + { + writeZerosToOutput( + b, + a.c_ptr, + stride_cm_block, + pid_n, + n_block, + offs_token, + token_mask, + block_size_m, + block_size_n, + compute_type, + ); + dead_ret.close(); + } + + // CSE `pid_n * BLOCK_SIZE_N` so it reuses across offs_bn / offs_cn. + const pid_n_block_n = pid_n.mul(block_size_n); + const offs_bn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)).rem(n_block); + + // offs_k stays i32 through expand_dims; extsi to i64 happens on the 2D form. + const offs_k_i32 = b.arange(0, block_size_k, .i32); + + // a_ptrs: `offs_token[:, None] // top_k * stride_am + offs_k[None, :]`. + // (stride_ak == 1 here, so `* stride_ak` folds away.) Build with explicit + // broadcastTo to match Python's emit shape. + const offs_token_col = b.expandDims(offs_token, 1); + const am_col = offs_token_col.div(top_k).mul(stride_am_block); + const offs_k_row = b.expandDims(offs_k_i32, 0).to(.i64); + const am_term = b.broadcastTo(am_col, &.{ block_size_m, block_size_k }); + const ak_term = b.broadcastTo(offs_k_row, &.{ block_size_m, block_size_k }); + const a_ptrs_init = a.a_ptr.addPtr(am_term.add(ak_term)); + + // b_ptrs: `b_ptr + off_experts * stride_be` (scalar addptr) THEN + // `+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)` (2D + // addptr) — matches Python's left-to-right evaluation. Defer extsi + // for offs_k to after offs_bn's expand_dims/mul (Python emit order). + const b_ptr_shifted = a.b_ptr.addPtr(off_experts.mul(stride_be_block)); + // MXFP4 packs two K values per byte; activation K remains unpacked. + const offs_bk_i32 = b.arange(0, block_size_bk, .i32); + const offs_k_col_i32 = b.expandDims(offs_bk_i32, 1); + const offs_bn_row = b.expandDims(offs_bn, 0); + const bn_row = offs_bn_row.mul(stride_bn_block); + const offs_k_col = offs_k_col_i32.to(.i64); + const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_bk, block_size_n }); + const bn_term = b.broadcastTo(bn_row, &.{ block_size_bk, block_size_n }); + const b_ptrs_init = b_ptr_shifted.addPtr(bk_term.add(bn_term)); + + const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); + + // Loop bounds stay i64 because `k_block` is i64. iter_args order + // matches Python's TTIR for the matrix and accumulator state. + const num_k_iters = k_block.cdiv(block_size_k); + var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); + { + const k_iter = loop.iv; + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + + const k_remaining = k_block.sub(k_iter.mul(block_size_k)); + + // mask_a = token_mask[:, None] & (offs_k[None, :] < k_remaining). + const token_mask_col_a = b.expandDims(token_mask, 1); + const offs_k_row_a = b.expandDims(offs_k_i32, 0).to(.i64); + const offs_k_lt_a = offs_k_row_a.lt(b.splat(k_remaining, &.{ 1, block_size_k })); + const token_mask_full_a = b.broadcastTo(token_mask_col_a, &.{ block_size_m, block_size_k }); + const offs_k_lt_a_full = b.broadcastTo(offs_k_lt_a, &.{ block_size_m, block_size_k }); + const mask_a = token_mask_full_a.bitAnd(offs_k_lt_a_full); + + const a_val = b.loadOpts(a_ptrs, .{ + .mask = mask_a, + .other = b.zeros(&.{ block_size_m, block_size_k }, cfg.a_dtype), + }); + + const offs_k_col_b = b.expandDims(offs_bk_i32, 1).to(.i64); + const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining.cdiv(weight_packing), &.{ block_size_bk, 1 })); + const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_bk, block_size_n }); + + const b_val = b.loadOpts(b_ptrs, .{ + .mask = mask_b, + .other = b.zeros(&.{ block_size_bk, block_size_n }, cfg.b_dtype), + }); + + const new_acc = if (cfg.quant_scheme) |scheme| switch (scheme) { + .mxfp4, .mxfp8 => scaled: { + const scale_k = @divExact(block_size_k, 32); + const groups = k_iter.mul(scale_k).add(b.arange(0, scale_k, .i64)); + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + // dot_scaled expects RHS scales in [N, K/32] order. + const rows = off_experts.mul(stride_bse).add(offs_bn.mul(stride_bsn)).expandDims(1); + const cols = groups.mul(stride_bsk).expandDims(0); + const offsets = b.broadcastTo(rows, &.{ block_size_n, scale_k }) + .add(b.broadcastTo(cols, &.{ block_size_n, scale_k })); + const n_mask = b.broadcastTo(offs_bn.lt(n_block).expandDims(1), &.{ block_size_n, scale_k }); + const k_mask = b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_n, scale_k }); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = n_mask.bitAnd(k_mask), + .other = b.full(&.{ block_size_n, scale_k }, 127, .i8), + }); + const a_scales: ?Value = if (cfg.a_scale_dtype != null) input_scales: { + const a_rows = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)).expandDims(1); + const a_cols = groups.mul(b.load(a.stride_ask_ptr)).expandDims(0); + const a_offsets = b.broadcastTo(a_rows, &.{ block_size_m, scale_k }) + .add(b.broadcastTo(a_cols, &.{ block_size_m, scale_k })); + const a_mask = b.broadcastTo(token_mask.expandDims(1), &.{ block_size_m, scale_k }) + .bitAnd(b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_m, scale_k })); + break :input_scales b.loadOpts(a.a_scale_ptr.addPtr(a_offsets), .{ + .mask = a_mask, + .other = b.full(&.{ block_size_m, scale_k }, 127, .i8), + }); + } else null; + break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, if (scheme == .mxfp4) .e2m1 else .e4m3, .{}); + }, + .fp8_per_channel, .fp8_per_tensor => b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }), + .fp8_block32, .fp8_block128 => scaled: { + // Activation scales are [token, K/block] and weight scales are + // [expert, N/block, K/block]. Compute their addresses from the + // loop index; activation scales are optional. + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + const b_scale_ptrs = a.b_scale_ptr.addPtr( + off_experts.mul(stride_bse) + .add(offs_bn.div(fp8_block_size).mul(stride_bsn)) + .add(k_iter.mul(stride_bsk)), + ); + const dot = b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), b.zeros(&.{ block_size_m, block_size_n }, .f32), .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + const b_s = b.loadOpts(b_scale_ptrs, .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype orelse .f32), + }).to(.f32); + var scaled_dot = dot.mul(b_s.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const a_scale_ptrs = a.a_scale_ptr.addPtr( + offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)) + .add(k_iter.mul(b.load(a.stride_ask_ptr))), + ); + const a_s = b.loadOpts(a_scale_ptrs, .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + scaled_dot = scaled_dot.mul(a_s.expandDims(1)); + } + break :scaled acc.add(scaled_dot); + }, + .nvfp4 => unreachable, + } else b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + + // BLOCK_SIZE_K is `tl.constexpr` → i32 dense splat (stride_ak == 1). + const bsk_i32: i32 = @intCast(cfg.block_size_k); + const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); + const new_b_ptrs = b_ptrs.addPtr(b.splat(@as(i32, @intCast(block_size_bk)), &.{ block_size_bk, block_size_n })); + + loop.yield(.{ + new_a_ptrs, + new_b_ptrs, + new_acc, + }); + } + var accumulator = loop.results[2]; + + if (cfg.quant_scheme == .fp8_per_channel or cfg.quant_scheme == .fp8_per_tensor) { + // These scales are constant along K, so apply them after accumulation. + const expert_offset = off_experts.mul(b.load(a.stride_bse_ptr)); + const offsets = if (cfg.quant_scheme == .fp8_per_channel) + expert_offset.add(offs_bn.mul(b.load(a.stride_bsn_ptr))) + else + b.splat(expert_offset, &.{block_size_n}); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype.?), + }).to(.f32); + accumulator = accumulator.mul(scales.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const offsets_a = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)); + const scales_a = b.loadOpts(a.a_scale_ptr.addPtr(offsets_a), .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + accumulator = accumulator.mul(scales_a.expandDims(1)); + } + } + + if (cfg.b_bias_dtype) |dtype| { + const offsets = off_experts.mul(b.load(a.stride_bbe_ptr)) + .add(offs_bn.mul(b.load(a.stride_bbn_ptr))); + const bias = b.loadOpts(a.b_bias_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, dtype), + }).to(.f32); + // Bias is in output units and must receive the routing weight too. + accumulator = accumulator.add(bias.expandDims(0)); + } + + if (cfg.routing_weights_dtype) |dtype| { + const other = b.zeros(&.{block_size_m}, dtype); + const routing_weights = b.loadOpts(a.routing_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = other }); + accumulator = accumulator.mul(routing_weights.to(.f32).expandDims(1)); + } + + accumulator = accumulator.to(compute_type); + + // c_ptrs: two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right eval. `stride_cm * offs_token[:, None]` puts the splatted + // stride on the LHS of arith.muli (Python source order). + const offs_cn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)); + const offs_token_col_c = b.expandDims(offs_token, 1); + const cm_col = stride_cm_block.mul(offs_token_col_c); + const c_ptrs_col = b.splat(a.c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n })); + + // c_mask = token_mask[:, None] & (offs_cn[None, :] < n_block). + const token_mask_col_c = b.expandDims(token_mask, 1); + const offs_cn_lt_2d = offs_cn_row.lt(b.splat(n_block, &.{ 1, block_size_n })); + const token_mask_full_c = b.broadcastTo(token_mask_col_c, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full_c.bitAnd(offs_cn_lt_full) }); + } +}; + +test "FP8 activation quantization emits float and E8M0 scales" { + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { + for ([_]DType{ .bf16, .f32 }) |input_dtype| { + const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ + .input_dtype = input_dtype, + .output_dtype = .f8e4m3fn, + .scale_dtype = if (scheme == .mxfp8) .i8 else .f32, + .block = if (scheme == .mxfp8 or scheme == .fp8_block32) 32 else 256, + .quant_scheme = scheme, + }); + defer std.testing.allocator.free(ir); + if (scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") != null); + } + } + } +} + +test "FusedMoe emits each FP8 and MXFP4 scaling path" { + const allocator = std.testing.allocator; + const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; + for (schemes) |scheme| { + for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { + if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; + for ([_]bool{ false, true }) |quantize_input| { + if ((scheme == null or scheme == .mxfp4) and quantize_input) continue; + for ([_]?DType{ null, .bf16, .f32 }) |bias_dtype| { + const ir = try FusedMoe.Kernel.emit(allocator, .{ + .a_dtype = if (quantize_input) fp8_dtype else .bf16, + .b_dtype = if (scheme == .mxfp4) .i8 else if (scheme != null) fp8_dtype else .bf16, + .c_dtype = .bf16, + .a_scale_dtype = if (quantize_input) (if (scheme == .mxfp8) .i8 else .f32) else null, + .b_scale_dtype = if (scheme == .mxfp4 or scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, + .b_bias_dtype = bias_dtype, + .routing_weights_dtype = .bf16, + .block_size_m = 16, + .block_size_n = 32, + .block_size_k = if (scheme == .fp8_block32) 32 else 128, + .group_size_m = 1, + .top_k = 2, + .naive_block_assignment = false, + .compute_type = .bf16, + .quant_scheme = scheme, + }); + defer allocator.free(ir); + if (scheme == .mxfp4 or scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") != null); + } else { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot") != null); + if (scheme != null and !quantize_input) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.fp_to_fp") != null); + } + } + } + } + } + } +} + +// ============================================================================= +// moe_align_block_size_kernel +// ============================================================================= + +/// Direct port of Python `moe_align_block_size_kernel`. pid==0 runs the +/// histogram + cumsum + block-to-expert assignment pass; pid==1 fills +/// sorted_token_ids with NUMEL. +/// +/// Block assignment are set up to `num_tokens_post_padded` and the remaining blocks of +/// `expert_ids` are not set to a sentinel value. +pub const MoeAlignBlockSize = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + padded_num_experts: usize, + max_num_tokens_padded: usize, + max_num_m_blocks: usize, + block_size_m: usize, + hist_block: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "moe_align_block_size_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_pad_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_pad_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + .out2_ptr = .{ .ptr = .i32 }, + .out3_ptr = .{ .ptr = .i32 }, + }); + + const block_size_m: i32 = @intCast(cfg.block_size_m); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const padded_num_experts: i64 = @intCast(cfg.padded_num_experts); + const max_num_tokens_padded: i32 = @intCast(cfg.max_num_tokens_padded); + const hist_block: i64 = @intCast(cfg.hist_block); + + const pid = b.programId(.x); + const fill_offs = b.arange(0, hist_block, .i32); + + // if pid == 1: fill sorted_token_ids with NUMEL; return + var fill_branch = b.openReturnIfOpts(pid.eq(1), .{ .inline_return = true }); + { + var fill_loop = b.openFor(0, max_num_tokens_padded, hist_block, .{}); + { + const offs = fill_loop.iv.add(fill_offs); + const mask = offs.lt(max_num_tokens_padded); + b.storeOpts( + a.sorted_token_ids_ptr.addPtr(offs), + b.splat(numel, &.{hist_block}), + .{ .mask = mask }, + ); + fill_loop.close(); + } + fill_branch.close(); + } + + // pid != 1: histogram + cumsum + block assignment. + const expert_offs = b.arange(0, padded_num_experts, .i32); + const token_offs = b.arange(0, hist_block, .i32); + const expert_mask = expert_offs.lt(num_experts); + const counts_init = b.zeros(&.{padded_num_experts}, .i32); + + var hist_loop = b.openFor(0, numel, hist_block, .{counts_init}); + { + const offs = hist_loop.iv.add(token_offs); + const mask = offs.lt(numel); + const expert_vals = b.loadOpts(a.topk_ids_ptr.addPtr(offs), .{ + .mask = mask, + .other = b.splat(num_experts, &.{hist_block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + const h = b.histogramOpts(expert_vals, padded_num_experts, .{ .mask = valid }); + hist_loop.yield(.{hist_loop.carried[0].add(h)}); + } + const counts = hist_loop.results[0]; + + // padded_counts = where(expert_mask, cdiv(counts, BLOCK_SIZE_M) * BLOCK_SIZE_M, 0) + const padded_counts = b.select( + expert_mask, + counts.cdiv(block_size_m).mul(block_size_m), + b.zeros(&.{padded_num_experts}, .i32), + ); + const padded_cumsum = b.cumsumOpts(padded_counts, .{ .axis = 0 }); + const starts = padded_cumsum.sub(padded_counts); + const total_tokens_post_pad = b.sumOpts(padded_counts, .{ .axis = 0 }); + + b.storeOpts(a.cumsum_ptr.addPtr(expert_offs), starts, .{ .mask = expert_mask }); + b.store(a.cumsum_ptr.addPtr(num_experts), total_tokens_post_pad); + b.store(a.num_tokens_post_pad_ptr, total_tokens_post_pad); + + // Each expert owns a disjoint padded range, including empty ranges. + // Scatter from register-resident prefixes; no cross-lane global reads. + const block_counts = padded_counts.div(block_size_m); + const max_blocks = b.maxOpts(block_counts, .{ .axis = 0 }); + var scatter = b.openFor(0, max_blocks, 1, .{}); + { + const block_ids = starts.div(block_size_m).add(scatter.iv); + const mask = expert_mask.bitAnd(block_counts.gt(scatter.iv)); + b.storeOpts(a.expert_ids_ptr.addPtr(block_ids), expert_offs, .{ .mask = mask }); + scatter.close(); + } + } +}; + +// ============================================================================= +// count_and_sort_expert_tokens_kernel +// ============================================================================= + +/// Direct port of Python `count_and_sort_expert_tokens_kernel`. Each program +/// atomically bumps `cumsum[expert]`, writing its block's token offsets into +/// `sorted_token_ids` at the returned rank. +pub const CountAndSortExpertTokens = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + sort_block_size: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "count_and_sort_expert_tokens_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + }); + + const block: i64 = @intCast(cfg.sort_block_size); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const block_i32: i32 = @intCast(block); + + const pid = b.programId(.x); + const num_progs = b.numPrograms(.x); + const token_offs = b.makeRange(0, block); + + const token_start_init = pid.mul(block_i32); + + // while token_start < NUMEL: + var w = b.openWhile(.{token_start_init}, .{.i32}); + { + const ts = w.before_carried[0]; + w.yieldBefore(ts.lt(numel), .{ts}); + } + { + const ts = w.after_carried[0]; + + // offs = ts + token_offs; mask = offs < NUMEL + const offs = ts.add(token_offs); + const mask = offs.lt(numel); + + // expert_vals = load(topk_ids + offs, mask=mask, other=NUM_EXPERTS) + const topk_ptrs = a.topk_ids_ptr.addPtr(offs); + const expert_vals = b.loadOpts(topk_ptrs, .{ + .mask = mask, + .other = b.splat(num_experts, &.{block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + + // rank = atomic_add(cumsum + expert_vals, 1, mask=valid, sem="relaxed") + const cumsum_ptrs = a.cumsum_ptr.addPtr(expert_vals); + const rank = b.atomicRmwOpts(.add, cumsum_ptrs, b.ones(&.{block}, .i32), .{ + .mask = valid, + .sem = .relaxed, + .scope = .gpu, + }); + + // store(sorted_token_ids + rank, offs, mask=valid) + const sorted_ptrs = a.sorted_token_ids_ptr.addPtr(rank); + b.storeOpts(sorted_ptrs, offs, .{ .mask = valid }); + + // token_start += num_programs * BLOCK_SIZE + // Compute step INSIDE the loop body (Python's source position). + // It's loop-invariant, so XLA's LICM hoists it out — the hoisted + // op-order then matches Python's IR (splats before step). + const step = num_progs.mul(block_i32); + w.yieldAfter(.{ts.add(step)}); + } + } +}; + +/// Returns the next power of two. +fn pow2(x: i64) i64 { + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); +} + +pub fn quantRows(tokens: i64) i64 { + return if (tokens >= 16) 16 else 1; +} + +fn ar(b: *Builder, n: i64) Value { + return b.arange(0, n, .i32); +} + +fn ld(b: *Builder, p: Value, mask: Value, dt: tri.DType, other: i64) Value { + return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.constant(dt, other) }); +} + +fn ci(b: *Builder, n: i64) Value { + return b.constant(.i32, n); +} + +fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { + const rows = x.shape().constSlice()[0]; + const cols = x.shape().constSlice()[1]; + const groups = @divExact(cols, 32); + const blocks = b.reshape(x, &.{ rows * groups, 32 }); + const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); + const scaled = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0); + // TODO(Corentin): We should gate that behind a platform check so that we don't do useless stuff on non-SM89 platforms. + // Find a way to add that to the builder, maybe a `b.platform()` or something like that. + const q = b: { + // SM89 lowers FP32 -> FP8 through FP16 with truncation. Round to odd at + // FP16 precision first so that this intermediate cannot turn a value just + // above an FP8 midpoint into an exact tie. Direct FP8 conversion is unchanged. + const scaled_bits = scaled.bitCast(.i32); + const sticky = b.select(scaled_bits.bitAnd(0x1fff).ne(0), b.full(scaled.shape().constSlice(), 0x2000, .i32), b.zeros(scaled.shape().constSlice(), .i32)); + const prepared = scaled_bits.bitAnd(-0x2000).bitOr(sticky).bitCast(.f32); + break :b prepared.to(.f8e4m3fn); + }; + const bits = b.shrui(sf.bitCast(.i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); + return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; +} + +pub fn combineBlock(tokens: i64, hidden: i64) i64 { + // Keep at least 512 CTAs before increasing per-CTA reduction work. + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + return 256; +} + +/// Computes an upper bound on the number of expert tiles needed. +/// When there is only one token, reserve a single tile per selected expert. +pub fn tiles(tokens: i64, topk: i64, experts: i64, bn: i64) i64 { + if (tokens == 1) return topk; + return @min(tokens * topk, @divTrunc(tokens * topk + experts * (bn - 1) + bn - 1, bn)); +} + +// ============================================================================= +// mxfp4_triton_quant +// ============================================================================= + +pub const Quant = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); + const rows = b.programId(.x).mul(quantRows(c.tokens)).add(ar(b, quantRows(c.tokens))); + const block = b.programId(.y); + const cols = block.mul(128).add(ar(b, 128)); + const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); + const mask = rows.lt(c.tokens).expandDims(1); + const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); + const q = quant32(b, x); + b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); + const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); + b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); + } +}; + +// ============================================================================= +// mxfp4_triton_route +// ============================================================================= + +pub const Route = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const e = b.programId(.x); + const p = ar(b, pow2(c.tokens * c.topk)); + const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); + const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); + b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); + b.store(a.counts.addPtr(e), valid.to(.i32).sum()); + } +}; + +// ============================================================================= +// mxfp4_triton_route_atomic +// ============================================================================= + +pub const RouteAtomic = struct { + pub const route_block = 1024; + + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = run }); + + /// Routing for large batches, used by `scheduleForCute`. + /// Each route takes the next free slot of its expert with an atomic + /// increment of `counts` (zero-initialized through `zeros`). Slots within an + /// expert are therefore unordered, which the grouped GEMMs do not care + /// about: every routed row is computed independently. + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); + const in_range = p.lt(c.tokens * c.topk); + const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); + const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); + const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); + b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); + } +}; + +// ============================================================================= +// mxfp4_triton_schedule +// ============================================================================= + +pub const Schedule = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + /// Expert groups the schedule may fill. + capacity: i64, + /// Routed rows per expert group, i.e. the consuming GEMM's tile N. + group: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = run }); + + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const nt = cfg.capacity; + // Route positions are i32 tensors; keep the group size in that type. + const group: i32 = @intCast(cfg.group); + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(128).add(ar(b, 128)); + const e = ld(b, a.ids.addPtr(p), p.lt(cfg.tokens * cfg.topk), .i32, -1).sub(ci(b, cfg.expert_offset)); + const es = ar(b, pow2(cfg.experts)); + const counts = ld(b, a.counts.addPtr(es), es.lt(cfg.experts), .i32, 0); + const ts = counts.cdiv(group); + const starts = ts.cumsum().sub(ts); + var first = b.openIf(b.programId(.x).eq(0)); + { + b.store(a.sched.addPtr(2 * nt), ts.sum()); + first.close(); + } + const live = p.lt(cfg.tokens * cfg.topk).bitAnd(e.ge(0)).bitAnd(e.lt(cfg.experts)); + const safe = e.maximum(0).minimum(pow2(cfg.experts) - 1); + const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); + const start = starts.gather(safe, 0); + const tile = start.add(pos.div(group)); + const row = pos.rem(group); + b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(cfg.tokens * cfg.topk) }); + b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); + b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); + b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); + } +}; + +// ============================================================================= +// mxfp4_triton_combine +// ============================================================================= + +pub const Combine = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + topk: i64, + expert_offset: i64, + experts: i64, + routing_weight_placement: RoutingWeightPlacement = .before_down, + bn: i64, + }; + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return combineWithBn(b, c); + } + + fn combineWithBn(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); + const t = b.programId(.x); + const block = combineBlock(c.tokens, c.hidden); + const h = b.programId(.y).mul(block).add(ar(b, block)); + var acc = b.zeros(&.{block}, .f32); + for (0..@intCast(c.topk)) |s| { + const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); + const row = if (c.tokens == 1) pair.mul(c.bn) else b.load(a.perm.addPtr(pair)); + const valid = if (c.tokens == 1) v: { + const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); + break :v e.ge(0).bitAnd(e.lt(c.experts)); + } else row.ge(0); + var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); + if (c.routing_weight_placement == .after_down) { + value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); + } + acc = acc.add(value); + } + b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); + } +}; + +// ============================================================================= +// mxfp4 gemm +// ============================================================================= + +pub const GateUp = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_gate_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .gate_up); + } +}; + +pub const Down = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .down); + } +}; + +const GemmConfig = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + scale_stride: i64, + routing_weight_placement: RoutingWeightPlacement = .before_down, + bn: i64, + bk: i64, +}; + +const Projection = enum { gate_up, down }; + +fn gemm(b: *Builder, c: GemmConfig, comptime projection: Projection) tri.FinishError!void { + const is_up = projection == .gate_up; + const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); + const block = b.programId(.x); + const tile = b.programId(.y); + const n = ar(b, c.bn); + const nt = tiles(c.tokens, c.topk, c.experts, c.bn); + const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); + const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); + var live = b.openIf(valid); + { + const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); + const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; + const width: i64 = if (is_up and c.tokens > 8) 256 else 128; + const ksize = if (is_up) c.hidden else c.intermediate; + const rows = if (is_up) 2 * c.intermediate else c.hidden; + const m = block.mul(width).add(ar(b, width)); + const k = ar(b, c.bk); + const ks = ar(b, @divExact(c.bk, 32)); + const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); + const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.constant(.i64, @divTrunc(ksize, 2)), b.constant(.i64, 1) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); + var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); + { + const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); + const kk = loop.iv.mul(c.bk).add(k); + const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); + const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); + const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); + var mask = kk.expandDims(0).lt(ksize); + var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); + // The small-batch epilogue leaves padded rows unwritten. + if (is_up or c.tokens <= 8) { + mask = mask.bitAnd(n.lt(count).expandDims(1)); + smask = smask.bitAnd(n.lt(count).expandDims(1)); + } + const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); + const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); + loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); + } + const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; + const out_n = ar(b, epilogue_n); + const acc = if (c.tokens == 1) live_row: { + // The remaining MMA columns are padding during single-token decode. + // Extract column zero before activation/reduction and output stores. + var v = loop.results[0]; + var cols: i64 = c.bn; + while (cols > 1) : (cols = @divTrunc(cols, 2)) { + v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; + } + break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); + } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); + if (is_up) { + const columns = @divExact(width, 2); + const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); + var g = halves[0]; + var u = halves[1]; + if (c.swiglu_limit > 0) { + g = g.minimum(c.swiglu_limit); + u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); + } + const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); + var mid = u.mul(b.divRn(g, ex.add(1.0))); + if (c.routing_weight_placement == .before_down) { + const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); + mid = mid.mul(rw); + } + // Compute scales and quantize directly from the FP32 SwiGLU result. + const q = quant32(b, mid); + const col = block.mul(columns).add(ar(b, columns)); + b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); + const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); + b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); + } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); + live.close(); + } +} diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig new file mode 100644 index 00000000..8ba799a5 --- /dev/null +++ b/zml/kernels/moe/triton/triton.zig @@ -0,0 +1,1072 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const toDType = zml.kernel.triton.from; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); + +const log = std.log.scoped(.@"zml/moe/triton"); + +fn numStages(config: LaunchConfig) i32 { + const platform = zml.Compiler.current().platform; + if (platform.target != .rocm) return config.num_stages; + + // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage + // reuses scratch space between operand layout conversions. + const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); + var stages = config.num_stages; + for (platform.devices) |device| { + const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); + const limit = if (attribute) |value| value.int64 else 64 * 1024; + stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); + } + return stages; +} + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + // HACK(Corentin): only rely on the scheme of gate_up for now. + + if (gate_up.quantization) |q| switch (q.scheme) { + .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { + return fusedExpertsMxfp4(input, routing, gate_up, down, options); + }, + else => {}, + }; + + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + return fusedExpertsBase(input, routing, gate_up, down, layout, options); +} + +pub fn fusedExpertsBase( + hidden_states: Tensor, + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + layout: ProjectionLayout, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "triton backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "triton backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "triton backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "triton backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "triton backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "triton backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "triton backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (expert_map_) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (expert_map_) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = gate_up_.bias, + .input_scale = input_scale, + .weight_scale = gate_up_.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = opts.activation.applyActivation(gate_up_out, layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = down_.bias, + .input_scale = down_input_scale, + .weight_scale = down_.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) Tensor { + // Native FP4 tensors expose logical K; the Triton operand is byte-packed. + const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); + stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); + if (opts.quant_scheme == .mxfp4) { + stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); + stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); + } + + if (opts.quant_scheme == .fp8_block32) { + const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); + stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); + stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); + stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); + } + + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); + for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { + stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); + } + const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); + const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); + const m_tokens = opts.input.dim(0); + const em_effective = if (m_tokens < block_size_m) + @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) + else + opts.routing.max_num_tokens_padded; + stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); + const grid_x = + @divExact(em_effective, block_size_m) * + (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); + + const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { + const scheme = opts.quant_scheme orelse break :blk scale; + break :blk switch (scheme) { + // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. + .mxfp4, .mxfp8 => scale.bitCast(.u8), + .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), + .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), + // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. + .fp8_block32 => scale.convert(.f32), + .fp8_block128, .nvfp4 => scale, + }; + } else null; + + const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; + const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; + const has_weight_scale_expert_stride = if (weight_scale) |scale| + !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) + else + false; + const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; + const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; + const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; + + return kernels.FusedMoe.Kernel.call( + .{ + .a_ptr = opts.input, + .b_ptr = weight, + .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), + .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), + .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), + .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), + .sorted_token_ids_ptr = opts.routing.sorted_token_ids, + .expert_ids_ptr = opts.expert_ids, + .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, + .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), + .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), + .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), + .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), + .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), + .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), + .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), + .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), + .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), + .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), + .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), + }, + .{ .c = opts.output_shape }, + .{ + .cfg = .{ + .a_dtype = toDType(opts.input.dtype()), + .b_dtype = toDType(weight.dtype()), + .c_dtype = toDType(opts.output_shape.dtype()), + .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, + .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, + .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, + .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, + .block_size_m = opts.launch_config.block_size_m, + .block_size_n = opts.launch_config.block_size_n, + .block_size_k = opts.launch_config.block_size_k, + .group_size_m = opts.launch_config.group_size_m, + .top_k = opts.top_k, + .naive_block_assignment = opts.routing.naive_block_assignment, + .compute_type = .bf16, + .quant_scheme = opts.quant_scheme, + }, + .grid = .{ @intCast(grid_x), 1, 1 }, + .num_warps = opts.launch_config.num_warps, + .num_stages = numStages(opts.launch_config), + }, + ).c; +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + const sparsity_factor: i64 = 4; + const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; + const max_num_tokens_padded = if (naive_block_assignment) + num_assignments * block_size_m + else if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else alignBlockSize(ids, num_experts, block_size_m); + + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { + const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_tokens = topk_ids_.dim(.token); + const topk = topk_ids_.dim(.topk); + const num_assignments = num_tokens * topk; + const max_num_tokens_padded = if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); + const warp_size: i64 = 32; + const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); + const sort_block_size: i64 = 256; + const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); + + const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); + var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); + var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); + var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); + var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); + + { + const align_outs = kernels.MoeAlignBlockSize.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .expert_ids_ptr = expert_ids, + .num_tokens_post_pad_ptr = num_tokens_post_padded, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .expert_ids = expert_ids.shape(), + .num_tokens_post_pad = num_tokens_post_padded.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .padded_num_experts = @intCast(padded_num_experts), + .max_num_tokens_padded = @intCast(max_num_tokens_padded), + .max_num_m_blocks = @intCast(max_num_m_blocks), + .block_size_m = @intCast(block_size_m), + .hist_block = 256, + }, + .grid = .{ 2, 1, 1 }, + .num_stages = 1, + .num_warps = 8, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .expert_ids = .expert_ids_ptr, + .num_tokens_post_pad = .num_tokens_post_pad_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = align_outs.sorted_token_ids; + expert_ids = align_outs.expert_ids; + num_tokens_post_padded = align_outs.num_tokens_post_pad; + cumsums = align_outs.cumsum; + } + + { + const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .sort_block_size = @intCast(sort_block_size), + }, + .grid = .{ @intCast(sort_grid_x), 1, 1 }, + .num_stages = 1, + .num_warps = 4, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = sort_outs.sorted_token_ids; + cumsums = sort_outs.cumsum; + } + + return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { + if (quantize) { + if (scheme) |scheme_| { + if (scheme_ != .mxfp4) { + const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); + return .{ input, scale }; + } + } + } + return .{ x.convert(.bf16), null }; +} + +fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { + const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { + .mxfp8 => .{ 32, .u8, 1e-10 }, + .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, + .fp8_block128 => .{ 128, .f32, 1e-10 }, + .fp8_block32 => .{ 32, .f32, 1e-4 }, + .mxfp4, .nvfp4 => unreachable, + }; + stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); + stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); + + const groups_per_row = @divExact(x.dim(1), group_size); + const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); + const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); + + const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( + .{ + .y_ptr = x, + .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), + .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), + }, + .{ .y_q = quantized.shape(), .y_s = scales.shape() }, + .{ + .cfg = .{ + .input_dtype = toDType(x.dtype()), + .output_dtype = toDType(output_dtype), + .scale_dtype = toDType(scale_dtype), + .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), + .quant_scheme = scheme, + }, + .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }, + ); + + return .{ outs.y_q, outs.y_s }; +} + +test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); + const gates = [_]f32{ 0.75, -1.25, 3.5 }; + const ups = [_]f32{ 0.875, -2.25, 4.5 }; + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); + values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } + } + } +} + +test "FP8 routed GEMM with bias matches dequantized weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn expandScales(scales: Tensor, shape: Shape) Tensor { + var expanded = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = @divExact(shape.dim(axis), expanded.dim(axis)); + const broad_shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(broad_shape, axes.dims()) + .reshape(expanded.shape().setDim(axis, shape.dim(axis))); + } + return expanded.withTags(shape.tags()); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + const weight = w.convert(.f8e4m3fn); + var input = x; + var input_scale: ?Tensor = null; + var reference_input = x.convert(.f32); + if (quantize_input) { + input, input_scale = prepareInput(x, scheme, .f8e4m3fn, true); + const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; + reference_input = input.convert(.f32).withTags(x.shape().tags()) + .mul(expandScales(scale.convert(.f32), x.shape())); + } + const weight_scale = if (scheme == .mxfp8) blk: { + const native = scales.convert(.f8e8m0); + break :blk if (mx_native) native else native.bitCast(.u8); + } else scales; + const config = launchConfigForTokens(8); + const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); + const actual = gemm(.{ + .input = input, + .weight = weight, + .bias = bias, + .input_scale = input_scale, + .weight_scale = weight_scale, + .routing_weights = weights, + .routing = routing, + .expert_ids = routing.expert_ids, + .launch_config = config, + .quant_scheme = scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), + }).squeeze(.topk); + const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); + const dot = reference_input.dot(selected_weight, .in); + const selected_bias = if (bias.rank() == 2) + bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) + else + bias; + const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); + const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); + return .{ .actual = actual, .expected = expected }; + } + }; + + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]bool{ false, true }) |quantize_input| { + for ([_]i64{ 1, 17 }) |tokens| { + const n: i64 = if (scheme == .fp8_block128) 256 else 48; + const scale_dims: [3]i64 = switch (scheme) { + .mxfp8 => .{ 8, n, 8 }, + .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, + .fp8_per_channel => .{ 8, n, 1 }, + .fp8_block128 => .{ 8, 2, 2 }, + else => unreachable, + }; + const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); + const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); + const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); + const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); + const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); + const bias: Tensor = if (tokens == 1) + .init(.{ .out = n }, .bf16) + else + .init(.{ .expert = 8, .out = n }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + switch (i) { + 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, + 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), + 5 => { + const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + }, + else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) + @as(f32, @floatFromInt(1 + j % 3)) / 4 + else + @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { + log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); + return err; + }; + } + } + } +} + +test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + const fp4 = storage_dtype != .bf16; + const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); + const is_gate = switch (layout) { + .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + }; + const dtype: DataType = if (fp4) .u8 else .bf16; + // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. + const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); + const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); + const gate_up_values = is_gate.select(gate_value, up_value); + var gate_up: zml.nn.Linear = .{ + .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), + .tag = Shape.toTag(.d), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), + } else null, + }; + var down: zml.nn.Linear = .{ + .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), + .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), + .tag = Shape.toTag(.dout), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), + } else null, + }; + if (storage_dtype == .f4e2m1) { + gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); + down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); + gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); + down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); + } + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) + .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); + return fusedExpertsBase(x, .{ + .ids = route.addConstant(2), + .weights = route.convert(.f32).addConstant(1).scale(0.25), + .global_num_experts = 8, + .local_num_experts = 8, + }, gate_up, down, layout, .{ + .routing_weight_placement = placement, + .quantize_input = false, + .activation = .{ .swiglu = .{} }, + }); + } + }; + for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { + const tokens: i64 = if (storage_dtype == .u8) 17 else 1; + const x: Tensor = .init(.{ .t = tokens, .d = 128 }, .bf16); + const host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); + defer exe.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; + // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. + const bias: f32 = if (placement == .before_down) 4 else 1.5; + const expected = 128 * activation * 0.75 + bias; + for (actual.constItems(zml.floats.BFloat16)) |value| { + try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); + } + } + } + } +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} + +test "MoE launch config selects the nearest token bucket" { + try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); + try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); + try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); + try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); + try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn validateOptions(opts: Options) void { + stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); +} + +pub const Config = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + global_experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. + routing_weight_placement: RoutingWeightPlacement = .before_down, + + pub fn validate(c: Config) !void { + if (c.tokens <= 0 or + c.experts <= 0 or + c.topk <= 0 or c.topk > c.global_experts or + c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) + { + return error.InvalidRoutingShape; + } + + if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or + c.intermediate < 128 or @mod(c.intermediate, 128) != 0) + { + return error.InvalidHiddenShape; + } + + if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { + return error.InvalidScale; + } + } +}; + +fn validateInputs( + c: Config, + input: zml.Tensor, + ids: zml.Tensor, + expert_weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, +) void { + stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); + + const shapes = .{ + .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), + .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), + .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), + .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), + .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), + }; + + stdx.debug.assert(input.shape().eql(shapes.input) and + ids.shape().eql(shapes.ids) and + expert_weights.shape().eql(shapes.expert_weights) and + // Note: we accept both i8 and u8 for gate_up and down + // TODO: we should instead use f4e2m1 + (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and + gate_up_scales.shape().eql(shapes.gate_up_scales) and + (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and + down_scales.shape().eql(shapes.down_scales), + \\Invalid MXFP4 tensor layout: + \\ - expected input={f}, got {f} + \\ - expected ids={f}, got {f} + \\ - expected expert_weights={f}, got {f} + \\ - expected gate_up={f}, got {f} + \\ - expected gate_up_scales={f}, got {f} + \\ - expected down={f}, got {f} + \\ - expected down_scales={f}, got {f} + , .{ + shapes.input, input, // + shapes.ids, ids, // + shapes.expert_weights, expert_weights, // + shapes.gate_up, gate_up, // + shapes.gate_up_scales, gate_up_scales, // + shapes.down, down, // + shapes.down_scales, down_scales, // + }); + + stdx.debug.assert( + gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and + down_scales.dim(2) >= @divTrunc(c.intermediate, 32), + "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", + .{ c, gate_up_scales, down_scales }, + ); +} + +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + + const x = input.reshape(.{ tokens, hidden }); + const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = routing.global_num_experts, + .topk = topk, + .swiglu_limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + }; + + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = gq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = dq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + return kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(input.shape().dims()).withTags(input.shape()); +} diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} diff --git a/zml/kernels/paged_attention/README.md b/zml/kernels/paged_attention/README.md new file mode 100644 index 00000000..20ba9d7d --- /dev/null +++ b/zml/kernels/paged_attention/README.md @@ -0,0 +1,184 @@ +# Paged attention semantics + +[paged_attention.zig](paged_attention.zig) evaluates attention over a KV cache +whose logical sequences are stored in fixed-size physical pages. Paging changes +how keys and values are addressed; softmax still normalizes over all permitted +keys in one sequence, independently for each query and query head. + +The public entry point is: + +```zig +pagedAttention(parameters, q, k, v, kv_cache, opts) +``` + +The result has the query's shape and dtype. The `k` and `v` arguments are currently +ignored: attention reads exclusively from `kv_cache`. Populate the cache, +including the current queries' keys and values, **before** calling attention. +The attention call does not update the cache or allocate pages. Projections, +RoPE, and any Q/K normalization happen before this operation; the output +projection happens afterward. + +## Tensor layouts and heads + +Let `T` be the packed query token count, `B` the sequence capacity, `N` the +physical page count, `P` the tokens per page, `M` the page-table capacity per +sequence, `Hkv` the KV head count, `G` the query heads per KV head, and `D` the +features per head. + +| Input | Tagged shape | Meaning | +| --- | --- | --- | +| `q` | `{ .b = T, .hkv = Hkv, .hg = G, .hd = D }` | Queries packed in sequence order. Here `.b` counts tokens, not sequences. | +| Split cache `k`, `v` | `{ .page = N, .k_chunk = P, .hkv = Hkv, .hd = D }` | Physical key/value storage. | +| `parameters.block_table` | `{ .b = B, .p = M }`, `i32` | Logical page to physical page mapping for each sequence. | +| `parameters.seq_lens` | `{ .b = B }`, `i32` | Valid KV token counts, including the current queries. | +| `parameters.query_start_len` | `{ .b = B + 1 }`, `i32` | Cumulative offsets into packed queries. | + +The total query head count is `Hq = Hkv * G`. Flattening the two head axes gives +`h = hkv * G + hg`, so query head `h` reads KV head `floor(h / G)`. Each query +head has its own scores and softmax. This covers MHA (`G = 1`), GQA, and MQA +(`Hkv = 1`). `D` is the feature count of one head, not the number of heads or +the sequence length. Use matching floating-point query and cache dtypes. + +The TPU backend instead requires `KvCache.dense`, a fused tensor with layout +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal `D` rounded up to a +multiple of 128. Pad unused cache features with zeros. The wrapper pads queries, +uses the logical `D` for the default scale, and trims the output back to `D`. +Its accepted query/cache dtypes are `bf16`, `f16`, and `f32`. + +## Packed sequences and positions + +For sequence `s`, define: + +```text +start = query_start_len[s] +end = query_start_len[s + 1] +Lq = end - start +Lk = seq_lens[s] +context_len = Lk - Lq +position(i) = context_len + i # 0 <= i < Lq +``` + +Queries `q[start:end]` represent the final `Lq` tokens of that sequence. The +kernel infers their positions from lengths; there is no separate position input. +An initial prefill has `Lq = Lk`, a chunked prefill has `Lq > 1` after a cached +prefix, and a decode normally has `Lq = 1` at position `Lk - 1`. A batch may mix +prefill and decode sequences. Set `is_prefill = true` for such mixed batches; +`false` selects the all-decode execution configuration on Triton. It does not +change the attention equation. `max_seqlen_q` is a configuration bound for the +largest query chunk, not a runtime sequence length. + +Offsets must begin at zero and be nondecreasing. Active sequences require +`0 < Lq <= Lk <= M * P`. Place inactive sequences after the active sequences, +with repeated offsets and normally `seq_lens = 0`. This ordering matters: +StableHLO stops at the first sequence with no queries, and TPU passes a count of +active sequences to its kernel. The final offset is the active packed token +count; any extra query capacity is padding whose output should not be consumed. + +## Page addressing and cache updates + +A logical key position `j` in sequence `s` is addressed as: + +```text +logical_page = floor(j / P) +physical_page = block_table[s, logical_page] +offset = j % P +K(s, j) = k_cache[physical_page, offset] +V(s, j) = v_cache[physical_page, offset] +``` + +Physical pages need not be contiguous or ordered. Different sequences may map +to the same physical pages to share a prefix. Every page covering a valid token +must have a valid physical ID in `[0, N)` and populated KV data. Entries beyond +`ceil(Lk / P)` are unused; tests use `-1` there. A negative ID is not a portable +way to mask a valid logical page. `seq_lens`, rather than allocated page count, +determines which tokens in the last page are valid. + +`KvCache.update(new_k, new_v, slot_mapping, chunk_size, backend)` writes split +cache entries using physical flattened slots: + +```text +chunk_size = P +physical_page = slot_mapping[t] / P +offset = slot_mapping[t] % P +``` + +`new_k` and `new_v` have packed token/head/feature axes matching the split +cache. `slot_mapping` is a physical address, whereas `block_table` translates +logical sequence positions. The caller maintains both mappings and lengths; +`update` does not derive or update sequence metadata. Writes override existing +entries and request buffer reuse. Use valid, distinct slots for writes requiring +deterministic results; conflicting scatter writes have no ordering guarantee. +The generic update method currently panics for `.dense`. + +## Attention equation and masks + +For local query row `i`, KV head `a`, and query group head `g`: + +```text +p = Lk - Lq + i +score_j = scale * dot(q[start + i, a, g], K(s, j, a)) +weight = softmax(score over permitted j) +out = sum_j(weight_j * V(s, j, a)) +``` + +`opts.scale` defaults to `1 / sqrt(D)`. There is no dropout or returned +probability matrix. Internal paging, tiling, or splitting of the key range must +preserve one softmax normalization; summing separately normalized page outputs +would compute a different result. Accumulation order and dtype conversions can +cause numerical differences between backends. + +The default mask is `.standard = .{ .causal = true }`: + +| Mask | Permitted logical key positions | +| --- | --- | +| Default causal | `0 <= j < Lk` and `j <= p`. Includes the query's own key. | +| `AttentionMask.none` | Every `0 <= j < Lk`. | +| Causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j <= p`. | +| Non-causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j`. Includes all later valid keys. | + +Use a positive `W`; it counts the current token as part of the left window. +Use `null` for no window. A non-causal window is a left bound, not a symmetric +window around the query. + +Triton also accepts `.indices`, an `i32` tensor shaped +`{ .b = T, .topk = K }` with `K > 0`. Each packed query row supplies logical +key positions in its own sequence, shared across its heads. Positions outside +`[0, Lk)` are ignored. This replaces the standard mask, so future valid keys +are permitted. Duplicate positions are processed repeatedly and therefore +change their weight; provide unique positions to represent a set of keys. + +On Triton, `opts.sink` supplies one softmax logit per flattened query head. +It adds `exp(sink[h])` to the denominator and contributes zero to the weighted +value numerator. It is not a cache token and does not consume a page or sequence +position. Other wrappers currently ignore `sink`. For portable use, ensure each +active query has at least one permitted key; fully masked rows are not given a +shared cross-backend output contract. + +## Example + +With `P = 4`, consider a two-query chunk after a four-token prefix and a decode +after a two-token prefix, followed by one inactive sequence: + +```text +query_start_len = [0, 2, 3, 3] +seq_lens = [6, 3, 0] +block_table = [[5, 2], [8, -1], [-1, -1]] +``` + +Packed queries 0 and 1 belong to sequence 0 at positions 4 and 5. Their current +KV entries occupy physical page 2, offsets 0 and 1 (slots 8 and 9). Packed query +2 belongs to sequence 1 at position 2, physical page 8, offset 2 (slot 34). +The default causal mask permits keys 0–4, 0–5, and 0–2 respectively, always +within the query's own sequence. With a window of 3, the permitted positions +become 2–4, 3–5, and 0–2. + +## Backend implementations + +See each backend's README for execution details, supported options, and current +limitations: + +- [Triton](triton/README.md) — CUDA, ROCm, and oneAPI. +- [Mosaic TPU](mosaic_tpu/README.md) — TPU. +- [Metal](metal/README.md) — Metal. +- [StableHLO](stablehlo/README.md) — CPU and Furiosa fallback. diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel new file mode 100644 index 00000000..6329ae12 --- /dev/null +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + "//zml/kernels/paged_attention/cute/sm10x", + "//zml/kernels/paged_attention/cute/sm120", + ], +) + +zig_test( + name = "correctness_test", + srcs = [ + "cases.zig", + "fixtures.zig", + "reference.zig", + ], + main = "correctness_test.zig", + tags = [ + "exclusive", + "requires-gpu", + ], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) + +zig_binary( + name = "benchmark", + srcs = [ + "cases.zig", + "fixtures.zig", + ], + main = "benchmark.zig", + tags = ["manual"], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig new file mode 100644 index 00000000..50eb2531 --- /dev/null +++ b/zml/kernels/paged_attention/cute/attention.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages: i64, + pages_per_seq: i64, + page_size: i64, + dtype: cute.DType, + causal: bool, + window: i64, + scale: f32, + sink: bool, + indexed: bool, + topk: i64, +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_generic", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *cute.Builder, c: Config) cute.FinishError!void { + switch (c.head_dim) { + 32 => try runDim(b, c, 1), + 64 => try runDim(b, c, 2), + 128 => try runDim(b, c, 4), + 256 => try runDim(b, c, 8), + 512 => try runDim(b, c, 16), + else => unreachable, + } +} + +fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + .k = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .v = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .table = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.batch, c.pages_per_seq } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.tokens, c.topk } } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + const lane = b.threadIdx().x; + const zero = b.constant(.f32, 0); + // One full warp owns a query/head; all control flow is warp-uniform. + // Also define output for unused capacity at the end of the packed queries. + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, zero.to(c.dtype)); + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find_seq = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find_seq.iv})).bitAnd(token.lt(a.starts.get(.{find_seq.iv.add(1)}))); + find_seq.yield(.{b.select(owns, find_seq.iv, find_seq.carried[0])}); + } + const seq = find_seq.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const len = a.lengths.get(.{seq}); + const position = len.sub(qlen).add(token.sub(start)); + const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) position.add(1).minimum(len) else len; + var query: [chunks]V = undefined; + for (&query, 0..) |*q, i| q.* = a.q.get(.{ token, head, lane.add(i * 32) }).to(.f32); + + // State is running max, normalizer, then each lane's output features. + // A sink is an initial softmax entry with zero value. + var initial: [chunks + 2]V = @splat(zero); + initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + initial[1] = if (c.sink) b.constant(.f32, 1) else zero; + var keys = b.openFor(if (c.indexed) b.constant(.i32, 0) else begin, if (c.indexed) b.constant(.i32, c.topk) else end, 1, initial); + { + const key = if (c.indexed) a.indices.get(.{ token, keys.iv }) else keys.iv; + const valid = key.ge(0).bitAnd(key.lt(len)); + // Invalid indexed entries must not load the page table, K, or V. Duplicates + // deliberately contribute once per occurrence, as independent softmax entries. + var entry = b.openIfElse(valid, @as([chunks + 2]V, @splat(zero))); + { + const page = a.table.get(.{ seq, key.div(c.page_size) }); + const offset = key.rem(c.page_size); + const kv_head = head.div(c.group); + var score = zero; + for (query, 0..) |q, i| score = score.add(q.mul(a.k.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); + score = score.mul(c.scale * std.math.log2e); + const maximum = keys.carried[0].maximum(score); + const alpha = b.exp2Approx(keys.carried[0].sub(maximum)); + const weight = b.exp2Approx(score.sub(maximum)); + var next: [chunks + 2]V = undefined; + next[0] = maximum; + next[1] = keys.carried[1].mul(alpha).add(weight); + for (0..chunks) |i| next[i + 2] = keys.carried[i + 2].mul(alpha).add(weight.mul(a.v.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + entry.yieldThen(next); + } + { + entry.yieldElse(keys.carried); + } + keys.yield(entry.results); + } + const norm = b.select(keys.results[1].gt(0), b.rcpApprox(keys.results[1]), zero); + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, keys.results[i + 2].mul(norm).to(c.dtype)); + active.close(); + } +} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig new file mode 100644 index 00000000..587af69f --- /dev/null +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +pub fn main(init: std.process.Init) !void { + const platform = try zml.Platform.auto(init.gpa, init.io, .{}); + defer platform.deinit(init.gpa, init.io); + + if (platform.target != .cuda) return error.CudaRequired; + + const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, + cc.minor, + platform.fmtVerbose(), + }); + + std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); + var failures: usize = 0; + for (cases.benchmarks) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + @tagName(c.mask), + }); + + runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + if (failures != 0) return error.BenchmarkFailed; +} + +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { + // Keep number of head per GPU constant. + var global_case = c; + global_case.kv_heads *= platform.devices.len; + var inputs = Inputs.init(global_case); + inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); + inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); + inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); + inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); + + const compile_start: std.Io.Timestamp = .now(io, .awake); + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ + .program_name = "paged_attention_harness", + .meshes = &.{mesh}, + }); + defer exe.deinit(); + const compile_us = elapsedUs(compile_start, io); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); + + var buffers = try fixtures.upload(io, platform, inputs, fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + + // Warmup + for (0..5) |_| { + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + output.deinit(); + } + + var samples: [101]f64 = undefined; + var sample_count: usize = 0; + var accumulated_us: f64 = 0; + while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { + const call_start: std.Io.Timestamp = .now(io, .awake); + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + samples[sample_count] = elapsedUs(call_start, io); + accumulated_us += samples[sample_count]; + sample_count += 1; + output.deinit(); + } + + std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); + const median = if (sample_count % 2 == 0) + (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 + else + samples[sample_count / 2]; + + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + compile_us / 1000, + sample_count, + median, + samples[sample_count / 10], + samples[(sample_count - 1) * 9 / 10], + }); +} + +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { + const Call = struct { + inputs: Inputs, + case: cases.Case, + + fn local(self: @This(), _: zml.Shape) zml.Tensor { + return fixtures.forward(self.inputs, self.case); + } + }; + const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + .in_specs = .{ .inputs = .{ + .q = heads, + .k = heads, + .v = heads, + .table = .replicated, + .lengths = .replicated, + .starts = .replicated, + .sinks = .parse(.{ ._0 = .model }), + .indices = .replicated, + } }, + .out_specs = heads, + }); +} + +fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { + return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; +} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig new file mode 100644 index 00000000..e8dedeaa --- /dev/null +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -0,0 +1,103 @@ +const std = @import("std"); + +pub const page_size = 16; +pub const topk = 8; +pub const Mask = enum { causal, non_causal, window, non_causal_window, indices }; + +pub const Case = struct { + name: []const u8 = "standard", + head_dim: usize = 128, + batch: usize = 2, + kv_heads: usize = 2, + group: usize = 4, + prefill: usize = 0, + chunk: usize = 17, + kv_len: usize = 65, + kv_capacity: ?usize = null, + inactive: usize = 0, + shuffle_pages: bool = false, + skew: bool = false, + mask: Mask = .causal, + empty_indices: bool = false, + window: u32 = 17, + sink: bool = false, + scale: ?f32 = null, + + pub fn tokens(c: Case) usize { + return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; + } + pub fn pages(c: Case) usize { + return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; + } + pub fn heads(c: Case) usize { + return c.kv_heads * c.group; + } + pub fn queryLen(c: Case, seq: usize) usize { + return if (seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; + } + pub fn kvLen(c: Case, seq: usize) usize { + const qlen = c.queryLen(seq); + if (qlen == 0) return 0; + const reduction = if (c.skew) (seq * 17) % c.kv_len else 0; + return @max(qlen, c.kv_len - @min(reduction, c.kv_len - 1)); + } +}; + +// Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. +pub const correctness = blk: { + var result: [34]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ .head_dim = dim, .prefill = if (mixed) 1 else 0 }; + i += 1; + } + } + result[8] = .{ .name = "mha_page_tail", .group = 1, .kv_len = 15 }; + result[9] = .{ .name = "mqa_page_exact", .kv_heads = 1, .group = 8, .kv_len = 16 }; + result[10] = .{ .name = "non_causal", .head_dim = 256, .prefill = 1, .mask = .non_causal }; + result[11] = .{ .name = "window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true, .scale = 0.125 }; + result[12] = .{ .name = "indices", .head_dim = 256, .prefill = 1, .mask = .indices, .sink = true }; + result[13] = .{ .name = "long_decode", .head_dim = 512, .kv_heads = 1, .group = 2, .batch = 1, .kv_len = 2049 }; + result[14] = .{ .name = "skew", .head_dim = 256, .batch = 4, .prefill = 2, .skew = true, .kv_len = 129 }; + result[15] = .{ .name = "mixed_short_chunks", .batch = 4, .prefill = 2, .chunk = 16, .skew = true, .kv_len = 64 }; + result[16] = .{ .name = "empty_indices", .prefill = 1, .mask = .indices, .empty_indices = true }; + result[17] = .{ .name = "d32", .head_dim = 32, .group = 1, .kv_len = 33 }; + result[18] = .{ .name = "odd_group", .group = 5, .prefill = 1, .chunk = 9, .kv_len = 79 }; + result[19] = .{ .name = "group_tile_tail", .head_dim = 256, .batch = 1, .group = 19, .kv_len = 257 }; + result[20] = .{ .name = "shuffled_mixed", .head_dim = 512, .batch = 4, .prefill = 2, .group = 3, .chunk = 11, .kv_len = 129, .shuffle_pages = true }; + result[21] = .{ .name = "split_sink", .head_dim = 512, .batch = 1, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[22] = .{ .name = "split_window_sink", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .window, .window = 257, .sink = true }; + result[23] = .{ .name = "inactive_sequence", .batch = 4, .prefill = 1, .inactive = 1, .kv_len = 129 }; + result[24] = .{ .name = "non_causal_window", .head_dim = 64, .prefill = 1, .kv_len = 65, .mask = .non_causal_window, .window = 7 }; + result[25] = .{ .name = "empty_splits", .batch = 1, .kv_len = 1, .kv_capacity = 4096, .sink = true }; + result[26] = .{ .name = "d32_mixed", .head_dim = 32, .group = 3, .prefill = 1, .chunk = 9, .kv_len = 257, .shuffle_pages = true }; + result[27] = .{ .name = "d64_group_tail", .head_dim = 64, .group = 19, .kv_len = 257 }; + result[28] = .{ .name = "split_mixed_sink", .head_dim = 128, .batch = 1, .prefill = 1, .chunk = 3, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[29] = .{ .name = "d512_prefill", .head_dim = 512, .batch = 2, .prefill = 2, .chunk = 65, .kv_len = 257, .sink = true, .shuffle_pages = true }; + result[30] = .{ .name = "negative_scale", .head_dim = 128, .prefill = 1, .kv_len = 257, .mask = .non_causal, .scale = -0.125 }; + result[31] = .{ .name = "zero_scale", .head_dim = 256, .prefill = 1, .kv_len = 257, .sink = true, .scale = 0 }; + result[32] = .{ .name = "d512_window_prefill", .head_dim = 512, .batch = 1, .prefill = 1, .chunk = 33, .kv_len = 513, .mask = .window, .window = 129, .sink = true }; + result[33] = .{ .name = "prefill_inactive", .head_dim = 64, .batch = 4, .prefill = 2, .inactive = 2, .kv_len = 257, .group = 5 }; + break :blk result; +}; + +pub const benchmarks = blk: { + var result: [24]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]usize{ 2, 8, 32 }) |batch| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ + .head_dim = dim, + .batch = batch, + .prefill = if (mixed) @max(1, batch / 4) else 0, + .chunk = 64, + .kv_len = 2048, + }; + i += 1; + } + } + } + break :blk result; +}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig new file mode 100644 index 00000000..436b9d3a --- /dev/null +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -0,0 +1,93 @@ +const reference = @import("reference.zig"); +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +test "CPU reference self checks" { + _ = @import("reference.zig"); +} + +test "paged attention correctness" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + + const a = std.testing.allocator; + const io = std.testing.io; + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, cc.minor, platform.fmtVerbose(), + }); + + var failures: usize = 0; + for (cases.correctness) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), + }); + + runCorrectness(a, io, platform, c, false) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + try std.testing.expectEqual(@as(usize, 0), failures); +} + +test "FP16 paged attention" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + for ([_]cases.Case{ + .{ .name = "fp16_decode", .head_dim = 64, .kv_len = 257 }, + .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, + .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, + .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, + }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); +} + +fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { + var inputs = i; + inputs.q = i.q.convert(.f16); + inputs.k = i.k.convert(.f16); + inputs.v = i.v.convert(.f16); + return fixtures.forward(inputs, c).convert(.bf16); +} + +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { + const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); + defer exe.deinit(); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), c); + if (compute_f16) { + // Exactly representable in both formats, so the existing CPU reference + // receives precisely the values used by the FP16 tensor-core path. + for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { + for (values) |*v| if (std.math.isFinite(v.toF32())) { + v.* = .fromF32(@round(v.toF32() * 64) / 64); + }; + } + } + + var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + + const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, + }); +} diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig new file mode 100644 index 00000000..ef78cc49 --- /dev/null +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -0,0 +1,115 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); +const sm10x = @import("sm10x/sm10x.zig"); +const sm120 = @import("sm120/sm120.zig"); + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return self.block_table.byteSize() + self.seq_lens.byteSize() + self.query_start_len.byteSize(); + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +// TODO(Corentin): This should be moved in the platform probably +pub fn multiprocessorCount(platform: *const zml.Platform) usize { + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); +} + +test { + _ = kernel; + _ = sm10x; + _ = sm120; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); + stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); + if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm10x.pagedAttention(parameters, q, k, v, opts); + if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm120.pagedAttention(parameters, q, k, v, opts); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages = k_.dim(.page), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), + .sink = opts.sink != null, + .indexed = opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig new file mode 100644 index 00000000..c0815d1d --- /dev/null +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -0,0 +1,141 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const cases = @import("cases.zig"); + +pub const Fixture = struct { + q: []zml.floats.BFloat16, + k: []zml.floats.BFloat16, + v: []zml.floats.BFloat16, + table: []i32, + lengths: []i32, + starts: []i32, + sinks: []f32, + indices: []i32, + + // The caller owns an arena; all arrays live until the case is complete. + pub fn init(a: std.mem.Allocator, c: cases.Case) !Fixture { + const f: Fixture = .{ + .q = try a.alloc(zml.floats.BFloat16, c.tokens() * c.heads() * c.head_dim), + .k = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .v = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .table = try a.alloc(i32, c.batch * c.pages()), + .lengths = try a.alloc(i32, c.batch), + .starts = try a.alloc(i32, c.batch + 1), + .sinks = try a.alloc(f32, c.heads()), + .indices = try a.alloc(i32, c.tokens() * cases.topk), + }; + var rng = std.Random.DefaultPrng.init(42); + const random = rng.random(); + for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; + @memset(f.indices, -1); + f.starts[0] = 0; + for (0..c.batch) |s| { + const len = c.kvLen(s); + f.lengths[s] = @intCast(len); + f.starts[s + 1] = f.starts[s] + @as(i32, @intCast(c.queryLen(s))); + for (0..c.pages()) |p| { + f.table[s * c.pages() + p] = if (p * cases.page_size < len) @intCast(s * c.pages() + p) else -1; + } + const live_pages = std.math.divCeil(usize, len, cases.page_size) catch unreachable; + if (c.shuffle_pages) random.shuffle(i32, f.table[s * c.pages() ..][0..live_pages]); + // Unused physical slots must not contribute to PV, even via zero * NaN. + for (len..c.pages() * cases.page_size) |j| { + const page = if (j / cases.page_size < live_pages) + @as(usize, @intCast(f.table[s * c.pages() + j / cases.page_size])) + else + s * c.pages() + j / cases.page_size; + const offset = (page * cases.page_size + j % cases.page_size) * c.kv_heads * c.head_dim; + @memset(f.k[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + @memset(f.v[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + } + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + const l: i32 = @intCast(len); + @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); + } + } + if (c.empty_indices) { + for (f.indices, 0..) |*index, i| index.* = if (i % 2 == 0) -1 else @intCast(c.kv_len); + @memset(f.k, .fromF32(std.math.nan(f32))); + @memset(f.v, .fromF32(std.math.nan(f32))); + } + return f; + } + + pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { + const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); + return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; + } +}; + +pub const Inputs = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + table: zml.Tensor, + lengths: zml.Tensor, + starts: zml.Tensor, + sinks: zml.Tensor, + indices: zml.Tensor, + + pub fn init(c: cases.Case) Inputs { + return .{ + .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), + .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), + .lengths = .init(.{ .b = c.batch }, .i32), + .starts = .init(.{ .b = c.batch + 1 }, .i32), + .sinks = .init(.{c.heads()}, .f32), + .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), + }; + } + + fn options(i: Inputs, c: cases.Case) zml.paged_attention.AttentionOptions { + return .{ + .mask = switch (c.mask) { + .causal => .{ .standard = .{} }, + .non_causal => .none, + .window => .{ .standard = .{ .sliding_window = c.window } }, + .non_causal_window => .{ .standard = .{ .causal = false, .sliding_window = c.window } }, + .indices => .{ .indices = i.indices }, + }, + .sink = if (c.sink) i.sinks else null, + .scale = c.scale, + }; + } +}; + +pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { + const parameters: zml.paged_attention.Parameters = .{ + .cute = .{ + .block_table = i.table, + .seq_lens = i.lengths, + .query_start_len = i.starts, + .options_ = .{ + .batch_size = c.batch, + .max_num_pages = c.pages(), + .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, + .is_prefill = c.prefill > 0, + }, + }, + }; + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); +} + +pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { + var result: zml.Bufferized(Inputs) = undefined; + var initialized: usize = 0; + errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { + if (index < initialized) @field(result, name).deinit(); + }; + inline for (@typeInfo(Inputs).@"struct".field_names) |name| { + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + initialized += 1; + } + return result; +} diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig new file mode 100644 index 00000000..1a030646 --- /dev/null +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -0,0 +1,127 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); + +/// One query/head, independent of GPU tensor operations. Duplicate indices are +/// separate softmax entries; invalid indices are excluded. +pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: usize, scores: []f32, out: []f32) void { + const len: usize = @intCast(f.lengths[seq]); + const start: usize = @intCast(f.starts[seq]); + const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); + const position = len - qlen + token - start; + const count = if (c.mask == .indices) cases.topk else len; + const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); + const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; + var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); + for (0..count) |j| { + const key = keyAt(c, f, token, j); + const valid = key >= 0 and key < len and switch (c.mask) { + .indices, .non_causal => true, + .causal => key <= position, + .window => key <= position and @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + .non_causal_window => @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + }; + scores[j] = -std.math.inf(f32); + if (!valid) continue; + const offset = f.kvOffset(c, seq, @intCast(key), head / c.group); + var dot: f32 = 0; + for (q, f.k[offset..][0..c.head_dim]) |qv, kv| dot += qv.toF32() * kv.toF32(); + scores[j] = dot * scale; + maximum = @max(maximum, scores[j]); + } + @memset(out, 0); + if (maximum == -std.math.inf(f32)) return; + var denominator: f32 = if (c.sink) @exp(f.sinks[head] - maximum) else 0; + for (0..count) |j| { + if (scores[j] == -std.math.inf(f32)) continue; + const weight = @exp(scores[j] - maximum); + denominator += weight; + const offset = f.kvOffset(c, seq, @intCast(keyAt(c, f, token, j)), head / c.group); + for (out, f.v[offset..][0..c.head_dim]) |*o, v| o.* += weight * v.toF32(); + } + for (out) |*v| v.* /= denominator; +} + +fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { + return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); +} + +pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { + const active_tokens: usize = @intCast(f.starts[c.batch]); + for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { + if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; + } + const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); + defer a.free(scores); + const expected = try a.alloc(f32, c.head_dim); + defer a.free(expected); + var max_error: f32 = 0; + var failures: usize = 0; + for (0..c.batch) |s| { + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + for (0..c.heads()) |h| { + row(c, f, s, t, h, scores, expected); + const base = (t * c.heads() + h) * c.head_dim; + for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { + const got = got_bf16.toF32(); + const err = @abs(want - got); + max_error = @max(max_error, err); + // Fixed BF16 error budget; never adapted to observed failures. + if (!std.math.isFinite(want) or err > 0.003 + 0.02 * @abs(want)) { + if (failures == 0) std.debug.print("Mismatch seq={d} token={d} head={d} dim={d}: expected={d} actual={d}\n", .{ s, t, h, d, want, got }); + failures += 1; + } + } + } + } + } + if (failures > 0) return error.ReferenceMismatch; + return max_error; +} + +test "reference single key, uniform causal scores, and sink" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [2]f32 = undefined; + var out: [64]f32 = undefined; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.sink = true; + f.sinks[0] = 0; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); +} + +test "reference windows, noncausal, duplicate and invalid indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [cases.topk]f32 = undefined; + var out: [64]f32 = undefined; + c.mask = .non_causal; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.mask = .window; + c.window = 1; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 4), v); + c.mask = .indices; + @memset(f.indices, -1); + @memcpy(f.indices[0..4], &[_]i32{ 0, 0, 1, 2 }); + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectApproxEqAbs(@as(f32, 8.0 / 3.0), v, 1e-6); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); +} diff --git a/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel new file mode 100644 index 00000000..05cd7edc --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm10x", + srcs = [ + "attention.zig", + "sm103.zig", + "sm10x.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig new file mode 100644 index 00000000..12d6273c --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -0,0 +1,667 @@ +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const sm103 = @import("sm103.zig"); + +pub const Architecture = enum { sm100, sm103 }; + +pub const Config = struct { + architecture: Architecture, + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + decode: bool, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return if (c.decode) (if (c.group <= 8) 8 else 16) else 64; + } + pub fn keys(c: Config) i64 { + return if (c.decode) 128 else 64; + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn paddedDim(c: Config) i64 { + return @max(128, c.head_dim); + } + fn features(c: Config) i64 { + return @divExact(c.paddedDim(), 128); + } + fn buffers(c: Config) i64 { + return if (c.decode) 2 else 3; + } + fn kvBytes(c: Config) i64 { + return c.keys() * 128 * 2; + } + fn qBase(c: Config) i64 { + return c.buffers() * c.kvBytes(); + } + fn pBase(c: Config) i64 { + return c.qBase() + c.rows() * c.paddedDim() * 2; + } + fn barriers(c: Config) i64 { + return c.pBase() + c.rows() * c.keys() * 2; + } + fn completion(c: Config) i64 { + return c.barriers() + c.buffers() * 16; + } + fn holder(c: Config) i64 { + return c.completion() + 16; + } + fn reductions(c: Config) i64 { + return c.holder() + 16; + } + fn sharedBytes(c: Config) i64 { + return c.reductions() + 4 * 16 * 4; + } + fn scoreColumn(c: Config) i64 { + return if (c.decode) c.features() * c.rows() else @min(c.paddedDim(), 448); + } + fn columns(c: Config) u32 { + return std.math.ceilPowerOfTwoAssert(u32, @intCast(@max(32, c.scoreColumn() + (if (c.decode) c.rows() else c.keys())))); + } +}; + +pub const block: [3]i32 = .{ 256, 1, 1 }; +pub const Attention = cute.Kernel(Config, .{ + .name = "paged_attention_cute", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = attention, +}); + +fn loadInt(b: *B, pointer: V, index: V) V { + return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// Each 64-feature plane uses a 128-byte XOR swizzle. The descriptor and +// vector copies use this same layout for Q, K, V and probabilities. +fn swizzle(row: V, col: V, rows: i64) V { + return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); +} + +fn descriptor(b: *B, address: V, leading_bytes: i64) V { + const bits: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | (@as(u64, @intCast(@divExact(leading_bytes, 16))) << 16); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(bits)))); +} + +fn instruction(c: Config, qk: bool) u32 { + const format: u32 = if (c.dtype == .bf16) 1 else 0; + const m: u32 = if (c.decode) 128 else 64; + const n: u32 = @intCast(if (c.decode) c.rows() else if (qk) c.keys() else 128); + return (1 << 4) | (format << 7) | (format << 10) | + (@as(u32, @intFromBool(c.decode and !qk)) << 15) | + (@as(u32, @intFromBool(!c.decode and !qk)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); +} + +const Range = struct { first: V, count: V }; + +// Used by both attention and merge: every output row must merge exactly the +// partitions produced for its query tile, including causal/window boundaries. +fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { + const first_pos = klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(klen) else klen; + const first = begin.div(c.keys()); + return .{ .first = first, .count = end.add(c.keys() - 1).div(c.keys()).sub(first).maximum(0) }; +} + +fn liveSplits(c: Config, tiles: V) V { + return tiles.add(1).div(2).maximum(1).minimum(c.splits); +} + +const Tile = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tmem: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + first_key_tile: V, + tiles: V, + split: V, + + fn ready(t: Tile, id: V) V { + return t.shared.add(t.c.barriers()).add(id.rem(t.c.buffers()).mul(8)); + } + fn released(t: Tile, id: V) V { + return t.ready(id).add(t.c.buffers() * 8); + } + fn phase(t: Tile, id: V) V { + return id.div(t.c.buffers()).rem(2); + } + fn valid(t: Tile, row: V, key: V) V { + const pos = t.klen.sub(t.qlen).add(row.div(t.c.group)); + var result = row.lt(t.qlen.mul(t.c.group)).bitAnd(key.lt(t.klen)); + if (t.c.causal) result = result.bitAnd(key.le(pos)); + if (t.c.window > 0) result = result.bitAnd(key.gt(pos.sub(t.c.window))); + return result; + } + fn putProbability(t: Tile, row: V, key: V, value: V) void { + t.b.storeHalf(t.shared.add(t.c.pBase()).add(swizzle(row, key, t.c.rows())), .shared, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + fn put(t: Tile, row: V, dim: V, value: V) void { + const token = t.start.add(row.div(t.c.group)); + var index = token.to(.i64).mul(t.c.kv_heads).add(t.head.to(.i64)).mul(t.c.group).add(row.rem(t.c.group).to(.i64)); + if (t.c.splits > 1) { + index = t.split.to(.i64).mul(t.c.tokens * t.c.kv_heads * t.c.group).add(index).mul(t.c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(index.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + index = index.mul(t.c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(index.mul(2)), .global, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + } + fn norm(t: Tile, row: V, maximum: V, denominator: V) V { + const b = t.b; + if (t.c.splits > 1) return b.constant(.f32, 1); + if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.constant(.f32, 0)); + const sink = loadInt(b, t.sink, t.head.mul(t.c.group).add(row.rem(t.c.group))).bitCast(.f32).mul(std.math.log2e); + const reference = maximum.maximum(sink).maximum(-std.math.floatMax(f32)); + const weight = b.exp2Approx(maximum.sub(reference)); + return weight.div(denominator.mul(weight).add(b.exp2Approx(sink.sub(reference)))); + } +}; + +fn attention(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .ptr = c.dtype }, + .k = .{ .ptr = c.dtype }, + .v = .{ .ptr = c.dtype }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, + }); + std.debug.assert(c.sharedBytes() <= 227 * 1024); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadInt(b, starts, seq); + const qlen = loadInt(b, starts, seq.add(1)).sub(start); + const klen = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); + const range = keyRange(b, c, first_row, qlen, klen); + const splits = liveSplits(c, range.count); + const split = b.blockIdx().z; + var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); + { + const tid = b.threadIdx().x; + const lo = range.count.mul(split).div(splits); + const hi = range.count.mul(split.add(1)).div(splits); + const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(c.sharedBytes()), 1024), .i32); + var t: Tile = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = shared, + .tmem = undefined, + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = klen, + .first_key_tile = range.first.add(lo), + .tiles = hi.sub(lo), + .split = split, + }; + var leader = b.openIf(tid.eq(0)); + { + for (0..@as(usize, @intCast(c.buffers()))) |i| { + b.mbarrierInit(shared.add(c.barriers() + @as(i64, @intCast(i * 8))), 128); + b.mbarrierInit(shared.add(c.barriers() + c.buffers() * 8 + @as(i64, @intCast(i * 8))), 1); + } + b.mbarrierInit(shared.add(c.completion()), 1); + b.fenceMbarrierInit(); + leader.close(); + } + var alloc = b.openIf(tid.lt(32)); + { + b.tmemAllocAt(shared.add(c.holder()), c.columns()); + b.relinquishTmemAllocPermit(.{}); + alloc.close(); + } + var queries = b.openFor(tid, c.rows() * @divExact(c.paddedDim(), 8), 256, .{}); + { + const row = queries.iv.div(@divExact(c.paddedDim(), 8)); + const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); + queries.close(); + } + b.cpAsyncWaitAll(); + b.syncThreads(); + t.tmem = b.loadWords(1, shared.add(c.holder()), .shared, .{})[0]; + b.tcgen05Fence(.after); + var producer = b.openIf(tid.ge(128)); + { + produce(t); + producer.close(); + } + var consumer = b.openIf(tid.lt(128)); + { + if (c.decode) { + if (c.group <= 2) decode(t, 2) else if (c.group <= 4) decode(t, 4) else if (c.group <= 8) decode(t, 8) else decode(t, 16); + } else prefill(t); + consumer.close(); + } + b.tcgen05Fence(.before); + b.syncThreads(); + var free = b.openIf(tid.lt(32)); + { + b.tmemDeallocAt(t.tmem, c.columns()); + free.close(); + } + active.close(); + } +} + +fn produce(t: Tile) void { + const b = t.b; + const c = t.c; + const row0 = t.tid.sub(128).div(16); + const col = t.tid.rem(16).mul(8); + var tiles = b.openFor(0, t.tiles, 1, .{}); + { + // Paged offsets are shared by K/V and every 128-feature stage. + var offsets: [16]V = undefined; + var valid: [16]V = undefined; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(i * 8); + valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); + const safe = b.select(key.lt(t.klen), key, b.constant(.i32, 0)); + const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); + offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); + } + for (0..@as(usize, @intCast(c.features() * 2))) |stage| { + const id = tiles.iv.mul(c.features() * 2).add(stage); + var reuse = b.openIf(id.ge(c.buffers())); + { + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); + reuse.close(); + } + const source = if (stage < c.features()) t.k else t.v; + const feature_offset: i64 = @mod(@as(i64, @intCast(stage)), c.features()) * 256; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const row = row0.add(i * 8); + b.cpAsync16(t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())).add(swizzle(row, col, c.keys())), source.add(offsets[i]).add(feature_offset), valid[i]); + } + b.cpAsyncMbarrierArrive(t.ready(id), true); + } + tiles.close(); + } +} + +fn multiply(t: Tile, tile: V, comptime qk: bool) void { + const b = t.b; + const c = t.c; + var leader = b.openIf(t.tid.eq(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); + b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); + b.tcgen05Fence(.after); + const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); + const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); + const steps: usize = @intCast(if (qk) 8 else @divExact(c.keys(), 16)); + for (0..steps) |step| { + const kv_offset: i64 = if (qk) @as(i64, @intCast(step / 4)) * c.keys() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step)) * 2048; + const qp_offset: i64 = (if (qk) @as(i64, @intCast(part * 2 + step / 4)) else @as(i64, @intCast(step / 4))) * c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; + const left = if (c.decode) kv.add(@divExact(kv_offset, 16)) else qp.add(@divExact(qp_offset, 16)); + const right = if (c.decode) qp.add(@divExact(qp_offset, 16)) else kv.add(@divExact(kv_offset, 16)); + const column: i64 = if (qk) c.scoreColumn() else @as(i64, @intCast(part)) * (if (c.decode) c.rows() else 128); + const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); + b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); + } + b.tcgen05Commit(t.released(id), .{}); + } + b.tcgen05Commit(t.shared.add(c.completion()), .{}); + leader.close(); + } + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); + b.tcgen05Fence(.after); + // A fast warp must not wrap completion parity before every consumer has + // observed the preceding PV completion. + if (!qk) b.barrierCtaSyncAligned(1, 128); +} + +fn publishProbabilities(t: Tile) void { + t.b.fenceProxyAsyncShared(); + t.b.tcgen05Fence(.before); + t.b.barrierCtaSyncAligned(1, 128); + t.b.tcgen05Fence(.after); +} + +fn reduceRow(comptime n: usize, values: [n]V, comptime maximum: bool) V { + var tree = values; + var width: usize = n; + while (width > 1) { + width /= 2; + for (0..width) |i| tree[i] = if (maximum) tree[i * 2].maximum(tree[i * 2 + 1]) else tree[i * 2].add(tree[i * 2 + 1]); + } + return tree[0]; +} + +fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximum: bool) [heads]V { + const b = t.b; + for (values, 0..) |value, h| { + var reduced = value; + inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); + var leader = b.openIf(t.lane.eq(0)); + { + b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(h * 4), .shared, .{reduced.bitCast(.i32)}); + leader.close(); + } + } + b.barrierCtaSyncAligned(1, 128); + var result: [heads]V = undefined; + for (0..heads) |h| { + var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); + for (1..4) |w| { + const next = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); + value = if (maximum) value.maximum(next) else value.add(next); + } + result[h] = value; + } + b.barrierCtaSyncAligned(1, 128); + return result; +} + +fn decode(t: Tile, comptime heads: usize) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const key_row = t.warp.mul(32).add(t.lane); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + var initial: [2 * heads]V = undefined; + for (0..heads) |h| { + initial[h] = neg_inf; + initial[heads + h] = zero; + } + var tiles = b.openFor(0, t.tiles, 1, initial); + { + multiply(t, tiles.iv, true); + const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); + var scores: [heads]V = undefined; + for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); + const maxima = reduceHeads(t, heads, scores, true); + var next: [2 * heads]V = undefined; + var alpha: [heads]V = undefined; + var probabilities: [heads]V = undefined; + for (0..heads) |h| { + next[h] = tiles.carried[h].maximum(maxima[h]); + const reference = next[h].maximum(-std.math.floatMax(f32)); + alpha[h] = b.exp2Approx(tiles.carried[h].sub(reference)); + probabilities[h] = b.exp2Approx(scores[h].sub(reference)); + } + const sums = reduceHeads(t, heads, probabilities, false); + for (0..heads) |h| { + next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); + t.putProbability(b.constant(.i32, h), key_row, probabilities[h]); + } + for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const addr = address.add(@as(i64, @intCast(part)) * c.rows()); + var out = b.tmemLoad32x32b(heads, addr); + b.fenceTmemLoad(); + for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + b.tmemStore32x32b(heads, addr, out); + } + b.fenceTmemStore(); + rescale.close(); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(next); + } + for (0..@as(usize, @intCast(c.features()))) |part| { + const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); + b.fenceTmemLoad(); + const dim = key_row.add(part * 128); + for (0..heads) |h| { + const row = t.first_row.add(h); + var valid = b.openIf(row.lt(t.qlen.mul(c.group))); + { + var feature = b.openIf(dim.lt(c.head_dim)); + { + const value = b.select(tiles.results[heads + h].gt(0), out[h].bitCast(.f32).mul(t.norm(row, tiles.results[h], tiles.results[heads + h])), zero); + t.put(row, dim, value); + feature.close(); + } + if (c.splits > 1 and part == 0) { + var stats = b.openIf(t.tid.eq(0)); + { + t.put(row, b.constant(.i32, c.head_dim), tiles.results[h]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[heads + h]); + stats.close(); + } + } + valid.close(); + } + } + } +} + +fn prefill(t: Tile) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const local_row = t.warp.mul(16).add(t.lane); + const row = t.first_row.add(local_row); + const owns = t.lane.lt(16); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + var tiles = b.openFor(0, t.tiles, 1, .{ neg_inf, zero }); + { + // D512 fills TMEM with O. Preserve its last 64 columns in registers + // while QK reuses them for scores, avoiding duplicate QK/softmax CTAs. + var saved: [64]V = undefined; + if (c.head_dim == 512) { + var preserve = b.openIfElse(tiles.iv.gt(0), @as([64]V, @splat(zero))); + { + const tail = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + var words: [64]V = undefined; + for (tail, 0..) |value, i| words[i] = value.bitCast(.f32); + preserve.yieldThen(words); + } + { + preserve.yieldElse(@as([64]V, @splat(zero))); + } + @memcpy(&saved, &preserve.results); + b.barrierCtaSyncAligned(1, 128); + } + multiply(t, tiles.iv, true); + var scores: [64]V = undefined; + var hardware_max = neg_inf; + if (c.architecture == .sm103) { + const loaded = sm103.loadScores(b, address.add(c.scoreColumn())); + scores = loaded.values; + hardware_max = loaded.maximum; + } else { + const loaded = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + for (loaded, 0..) |value, i| scores[i] = value.bitCast(.f32); + } + const key0 = t.first_key_tile.add(tiles.iv).mul(c.keys()); + // Hardware maxima describe raw scores. Only fully visible tiles may + // use them; page tails and mask boundaries reduce post-mask values. + const full = owns.bitAnd(t.valid(row, key0)).bitAnd(t.valid(row, key0.add(63))); + for (0..64) |i| { + scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(i))), scores[i].mul(c.scale * std.math.log2e), neg_inf); + } + const tile_max = if (c.architecture == .sm103 and c.scale > 0) blk: { + var choose = b.openIfElse(full, .{zero}); + { + choose.yieldThen(.{hardware_max.mul(c.scale * std.math.log2e)}); + } + { + choose.yieldElse(.{reduceRow(64, scores, true)}); + } + break :blk choose.results[0]; + } else reduceRow(64, scores, true); + const maximum = tiles.carried[0].maximum(tile_max); + const reference = maximum.maximum(-std.math.floatMax(f32)); + const alpha = b.exp2Approx(tiles.carried[0].sub(reference)); + var probabilities: [64]V = undefined; + for (0..64) |i| { + probabilities[i] = b.exp2Approx(scores[i].sub(reference)); + } + const denominator = tiles.carried[1].mul(alpha).add(reduceRow(64, probabilities, false)); + var write = b.openIf(owns); + { + for (0..64) |i| t.putProbability(local_row, b.constant(.i32, i), probabilities[i]); + write.close(); + } + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(@divExact(if (c.head_dim == 512) c.scoreColumn() else c.paddedDim(), 32)))) |chunk| { + const addr = address.add(chunk * 32); + var out = b.tmemLoad32x32b(32, addr); + b.fenceTmemLoad(); + for (&out) |*value| value.* = value.bitCast(.f32).mul(alpha).bitCast(.i32); + b.tmemStore32x32b(32, addr, out); + } + if (c.head_dim == 512) { + for (&saved) |*value| value.* = value.mul(alpha).bitCast(.i32); + b.tmemStore32x32b(64, address.add(c.scoreColumn()), saved); + } + b.fenceTmemStore(); + rescale.close(); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(.{ maximum, denominator }); + } + const norm = t.norm(row, tiles.results[0], tiles.results[1]); + for (0..@as(usize, @intCast(@divExact(c.paddedDim(), 32)))) |chunk| { + const out = b.tmemLoad32x32b(32, address.add(chunk * 32)); + b.fenceTmemLoad(); + var valid = b.openIf(owns.bitAnd(row.lt(t.qlen.mul(c.group)))); + { + for (0..32) |i| { + const dim = chunk * 32 + i; + if (dim < c.head_dim) t.put(row, b.constant(.i32, dim), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); + } + if (c.splits > 1 and chunk == 0) { + t.put(row, b.constant(.i32, c.head_dim), tiles.results[0]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[1]); + } + valid.close(); + } + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_merge", + .inputs = &.{ "partials", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{if (c.sink) c.kv_heads * c.group else 1} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); + find.yield(.{b.select(owns, find.iv, find.carried[0])}); + } + const seq = find.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const row = token.sub(start).mul(c.group).add(head.rem(c.group)); + const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); + const splits = liveSplits(c, range.count); + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const reference = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(reference)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(reference)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + const norm = b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero); + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(norm).to(c.dtype)); + dims.close(); + } + active.close(); + } +} + +test "SM10x layouts and instruction feature gates" { + for ([_]i64{ 32, 64, 128, 256, 512 }) |dim| for ([_]bool{ true, false }) |decode_only| for ([_]Architecture{ .sm100, .sm103 }) |architecture| { + const c: Config = .{ .architecture = architecture, .head_dim = dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 5, .pages_per_seq = 256, .page_size = 16, .max_query = if (decode_only) 1 else 17, .dtype = .bf16, .decode = decode_only, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 4 }; + try std.testing.expect(c.columns() >= 32 and c.columns() <= 512); + try std.testing.expect(c.sharedBytes() <= 227 * 1024); + const ir = try Attention.emit(std.testing.allocator, c, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); + try std.testing.expectEqual(architecture == .sm103 and !decode_only, std.mem.indexOf(u8, ir, "tcgen05.ld.red") != null); + const merge_ir = try Merge.emit(std.testing.allocator, c, .{ 128, 1, 1 }); + defer std.testing.allocator.free(merge_ir); + }; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig new file mode 100644 index 00000000..fe96916d --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -0,0 +1,36 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); + +const cute = @import("../../../../zml.zig").kernel.cute; + +pub const Scores = struct { + values: [64]cute.Value, + maximum: cute.Value, +}; + +// SM103 adds a per-TMEM-lane reduction across the loaded columns. This is +// useful for query-major scores; swapped decode reduces across lanes instead. +pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { + const asm_parts = comptime blk: { + @setEvalBranchQuota(20000); + var registers: []const u8 = ""; + var constraints: []const u8 = ""; + for (0..65) |i| { + if (i < 64) registers = registers ++ (if (i == 0) "" else ", ") ++ std.fmt.comptimePrint("${d}", .{i}); + constraints = constraints ++ (if (i == 0) "" else ",") ++ "=f"; + } + break :blk .{ + "tcgen05.ld.red.sync.aligned.32x32b.x64.max.f32 {" ++ registers ++ "}, $64, [$65];", + constraints ++ ",r", + }; + }; + const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([65]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); + var scores: Scores = undefined; + for (0..65) |i| { + const value = b.emit(llvm.extractvalue(b.ctx, result.inner, cute.DType.f32.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + if (i < 64) scores.values[i] = value else scores.maximum = value; + } + b.fenceTmemLoad(); + return scores; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig new file mode 100644 index 00000000..afc4298e --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const cute = @import("../cute.zig"); +const kernel = @import("attention.zig"); + +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .decode = decode, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); +} + +const ArchitectureCheck = struct { + fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { + const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + const table = zml.Tensor.constantTensor(.init(.{ 1, 9 }, .i32), std.mem.asBytes(&table_values)); + const lengths = zml.Tensor.constantTensor(.init(.{1}, .i32), std.mem.asBytes(&[_]i32{129})); + const starts = zml.Tensor.constantTensor(.init(.{2}, .i32), std.mem.asBytes(&[_]i32{ 0, 3 })); + const sink = zml.Tensor.zeroes(.init(.{3}, .f32)); + const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink }, .{ + .out = .init(.{ 2, 3, 3, c.head_dim + 2 }, .f32), + }, .{ .cfg = c, .grid = .{ 1, 1, 2 }, .block = kernel.block }).out; + return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ 3, 3, 1 }, + .block = .{ 128, 1, 1 }, + }).out; + } +}; + +test "SM10x GB200 and GB300 instruction paths match CPU reference" { + const platform = zml.testing.env(); + if (!isAvailable(platform)) return error.SkipZigTest; + const cc = zml.platform.cuda.computeCapability(platform).?; + const a = std.testing.allocator; + const io = std.testing.io; + const BF16 = zml.floats.BFloat16; + const table: [9]usize = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + for ([_]i64{ 128, 512 }) |head_dim| { + const d: usize = @intCast(head_dim); + const q_host = try a.alloc(BF16, 9 * d); + defer a.free(q_host); + const k_host = try a.alloc(BF16, 144 * d); + defer a.free(k_host); + const v_host = try a.alloc(BF16, 144 * d); + defer a.free(v_host); + for (q_host, 0..) |*value, i| value.* = .fromF32(@as(f32, @floatFromInt(i % 13)) / 16 - 0.375); + @memset(k_host, .fromF32(std.math.nan(f32))); + @memset(v_host, .fromF32(std.math.nan(f32))); + for (0..129) |key| for (0..d) |feature| { + const offset = (table[key / 16] * 16 + key % 16) * d + feature; + k_host[offset] = .fromF32(@as(f32, @floatFromInt((key * 7 + feature) % 17)) / 16 - 0.5); + v_host[offset] = .fromF32(@as(f32, @floatFromInt((key + feature * 3) % 23)) / 16 - 0.625); + }; + const q: zml.Tensor = .init(.{ .b = 3, .hkv = 1, .hg = 3, .hd = d }, .bf16); + const k: zml.Tensor = .init(.{ .page = 9, .k_chunk = 16, .hkv = 1, .hd = d }, .bf16); + const v: zml.Tensor = .fromShape(k.shape()); + var qb = try zml.Buffer.fromBytes(io, platform, q.shape(), std.mem.sliceAsBytes(q_host)); + defer qb.deinit(); + var kb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(k_host)); + defer kb.deinit(); + var vb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(v_host)); + defer vb.deinit(); + for ([_]kernel.Architecture{ .sm100, .sm103 }) |architecture| { + if (architecture == .sm103 and cc.minor < 3) continue; + const c: kernel.Config = .{ .architecture = architecture, .head_dim = head_dim, .tokens = 3, .batch = 1, .kv_heads = 1, .group = 3, .pages_per_seq = 9, .page_size = 16, .max_query = 3, .dtype = .bf16, .decode = false, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 2 }; + const exe = try platform.compileFn(a, io, ArchitectureCheck.forward, .{ q, k, v, c }, .{}); + defer exe.deinit(); + var output = try exe.eval(a, io, .{ qb, kb, vb }); + defer output.deinit(); + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + const actual = host.items(BF16); + for (0..9) |row| { + const keys = 127 + row / 3; + var scores: [129]f32 = undefined; + var maximum: f32 = 0; // Sink score, with zero value. + for (0..keys) |key| { + const offset = (table[key / 16] * 16 + key % 16) * d; + var score: f32 = 0; + for (0..d) |feature| score += q_host[row * d + feature].toF32() * k_host[offset + feature].toF32(); + scores[key] = score * c.scale; + maximum = @max(maximum, scores[key]); + } + var denominator = @exp(-maximum); + for (0..keys) |key| { + scores[key] = @exp(scores[key] - maximum); + denominator += scores[key]; + } + for (0..d) |feature| { + var expected: f32 = 0; + for (0..keys) |key| expected += scores[key] * v_host[(table[key / 16] * 16 + key % 16) * d + feature].toF32(); + expected /= denominator; + const got = actual[row * d + feature].toF32(); + try std.testing.expect(std.math.isFinite(got)); + try std.testing.expect(@abs(got - expected) <= 0.003 + 0.02 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/BUILD.bazel b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel new file mode 100644 index 00000000..f93bd5e0 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm120", + srcs = [ + "attention.zig", + "sm120.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig new file mode 100644 index 00000000..e9675252 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -0,0 +1,446 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + warps_per_query: i64, + kv_tile: i64, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return @divExact(64, c.warps_per_query); + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn kvBase(c: Config) i64 { + return c.rows() * c.head_dim * 2; + } + fn pBase(c: Config) i64 { + return c.kvBase() + c.kv_tile * c.head_dim * 2; + } + fn redBase(c: Config) i64 { + return c.pBase() + c.rows() * c.kv_tile * 2; + } + fn sharedBytes(c: Config) i64 { + return c.redBase() + 2 * 64 * 4; + } +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { + var result: [n * 4]V = undefined; + for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); + return result; +} + +fn packAcc(b: *B, values: [4]V) V { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn loadI32(b: *B, ptr: V, index: V) V { + return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. +fn swizzle(row: V, col: V, width: i64) V { + return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); +} + +fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { + var result: [n]V = undefined; + for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + return result; +} + +fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { + const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ + (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); + return unpackStruct(b, n, result, .i32); +} + +fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +fn run(b: *B, c: Config) cute.FinishError!void { + switch (c.head_dim) { + inline 32, 64, 128, 256, 512 => |d| switch (c.warps_per_query) { + inline 2, 4 => |w| switch (c.kv_tile) { + inline 32, 64 => |n| switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } +} + +const TileContext = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + split: V, + + // Every load resolves its logical page on device. Tail copies are zeroed + // before either MMA, including V tails whose backing allocation holds NaN. + fn loadKV(t: TileContext, source: V, key_base: V) void { + const b = t.b; + const c = t.c; + if (c.page_size == 16 and c.head_dim >= 64) { + // A page ID is shared by all feature vectors in its 16 rows. + // Resolve it once, then issue independent vector copies instead + // of serializing a page-table load before every cp.async. + var pages = b.openFor(0, c.kv_tile, 16, .{}); + { + const first_key = key_base.add(pages.iv); + const page_valid = first_key.lt(t.klen); + const logical_page = b.select(page_valid, first_key.div(16), b.constant(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(logical_page)); + const base = page.to(.i64).mul(16 * c.kv_heads * c.head_dim).add(t.head.to(.i64).mul(c.head_dim)); + for (0..@as(usize, @intCast(@divExact(c.head_dim, 64)))) |i| { + const vector_index = t.tid.add(i * 128); + const row = vector_index.div(@divExact(c.head_dim, 8)); + const col = vector_index.rem(@divExact(c.head_dim, 8)).mul(8); + const offset = base.add(row.to(.i64).mul(c.kv_heads * c.head_dim)).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(pages.iv.add(row), col, c.head_dim)), source.add(offset.mul(2)), first_key.add(row).lt(t.klen)); + } + pages.close(); + } + return; + } + var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), 128, .{}); + { + const row = copy.iv.div(@divExact(c.head_dim, 8)); + const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); + const key = key_base.add(row); + const valid = key.lt(t.klen); + const safe_key = b.select(valid, key, b.constant(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(safe_key.div(c.page_size))); + const offset = page.to(.i64).mul(c.page_size).add(safe_key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + copy.close(); + } + } + + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { + const b = t.b; + var v = value; + inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); + const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); + var leader = b.openIf(t.lane.rem(4).eq(0)); + { + b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); + leader.close(); + } + // Callers synchronize once both row halves have been published. + } + + fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { + const b = t.b; + const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); + var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); + for (1..w) |i| { + const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); + result = if (maximum) result.maximum(v) else result.add(v); + } + return result; + } + + fn store(t: TileContext, row: V, dim: V, value: V) void { + const c = t.c; + const token = t.start.add(row.div(c.group)); + var offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.rem(c.group).to(.i64)); + if (c.splits > 1) { + offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + offset = offset.mul(c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); + } + } +}; + +fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { + std.debug.assert(c.sharedBytes() <= 99 * 1024); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadI32(b, starts, seq); + const qlen = loadI32(b, starts, seq.add(1)).sub(start); + var active = b.openIf(first_row.lt(qlen.mul(c.group))); + { + const tid = b.threadIdx().x; + const t: TileContext = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), + .split = b.blockIdx().z, + }; + var copy_q = b.openFor(tid, c.rows() * (d / 8), 128, .{}); + { + const row = copy_q.iv.div(d / 8); + const col = copy_q.iv.rem(d / 8).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(d).add(col.to(.i64)); + b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); + copy_q.close(); + } + const first_pos = t.klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = t.klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(t.klen) else t.klen; + const first_tile = begin.div(n); + const tiles = end.add(n - 1).div(n).sub(first_tile).maximum(0); + const lo = first_tile.add(tiles.mul(t.split).div(c.splits)); + const hi = first_tile.add(tiles.mul(t.split.add(1)).div(c.splits)); + const row_base = t.warp.div(w).mul(16); + const key_part = t.warp.rem(w).mul(n / w); + const dim_part = t.warp.rem(w).mul(d / w); + const row0 = t.lane.div(4); + const col0 = t.lane.rem(4).mul(2); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const nf = n / w / 8; + const of = d / w / 8; + const count = 4 + of; + var initial: [count]V = @splat(zero); + initial[0] = neg_inf; + initial[1] = neg_inf; + for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); + b.cpAsyncWaitAll(); + b.syncThreads(); + var loop = b.openFor(lo, hi, 1, initial); + { + const key_base = loop.iv.mul(n); + t.loadKV(t.k, key_base); + b.cpAsyncWaitAll(); + b.syncThreads(); + var scores: [nf][4]V = @splat(@splat(zero)); + // Keep only MMA operands live in registers; Q remains in shared + // memory so D512 does not consume the entire per-thread register file. + var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); + { + const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); + for (0..nf) |i| { + const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); + scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); + } + qk.yield(flatten(nf, scores)); + } + for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; + var local_max: [2]V = @splat(neg_inf); + for (0..nf) |i| { + for (0..4) |j| { + const row = first_row.add(row_base).add(row0).add((j / 2) * 8); + const position = t.klen.sub(qlen).add(row.div(c.group)); + const key = key_base.add(key_part).add(i * 8).add(col0).add(j % 2); + var valid = row.lt(qlen.mul(c.group)).bitAnd(key.lt(t.klen)); + if (c.causal) valid = valid.bitAnd(key.le(position)); + if (c.window > 0) valid = valid.bitAnd(key.gt(position.sub(c.window))); + scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); + local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); + b.syncThreads(); + // All K reads have completed. Reuse its allocation for V, and + // overlap these global loads with softmax and probability stores. + t.loadKV(t.v, key_base); + var m: [2]V = undefined; + var alpha: [2]V = undefined; + var local_sum: [2]V = @splat(zero); + for (0..2) |h| { + m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); + // Empty/fully masked row tiles have m=-inf and denominator 0. + alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); + } + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + for (0..2) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + b.cpAsyncWaitAll(); + b.syncThreads(); + var next: [count]V = undefined; + for (0..2) |h| { + next[h] = m[h]; + next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); + } + var acc: [of][4]V = undefined; + for (0..of) |i| for (0..4) |j| { + acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); + }; + var packed_acc: [of]V = undefined; + for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); + var pv = b.openFor(0, n, 16, packed_acc); + { + const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + for (0..of) |i| { + const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); + var prev: [4]V = undefined; + for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); + acc[i] = mma(b, dt, pa, vb, prev); + packed_acc[i] = packAcc(b, acc[i]); + } + pv.yield(packed_acc); + } + @memcpy(next[4..], &pv.results); + b.syncThreads(); + loop.yield(next); + } + for (0..2) |h| { + const row = first_row.add(row_base).add(row0).add(h * 8); + var valid = b.openIf(row.lt(qlen.mul(c.group))); + { + const m = loop.results[h]; + const l = loop.results[h + 2]; + var norm = b.select(l.gt(0), b.rcpApprox(l), zero); + if (c.splits == 1 and c.sink) { + const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row.rem(c.group))).bitCast(.f32).mul(std.math.log2e); + const maximum = m.maximum(sink); + const weight = b.exp2Approx(m.sub(maximum)); + norm = weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); + } + for (0..of) |i| for (0..2) |j| { + const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); + t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); + }; + if (c.splits > 1) { + var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); + { + t.store(row, b.constant(.i32, d), m); + t.store(row, b.constant(.i32, d + 1), l); + leader.close(); + } + } + valid.close(); + } + } + active.close(); + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120_merge", + .inputs = &.{ "partials", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, c.splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const m = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, c.splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); + dims.close(); + } + valid.close(); + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig new file mode 100644 index 00000000..6645abf1 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -0,0 +1,69 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const cute = @import("../cute.zig"); +const kernel = @import("attention.zig"); + +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 12; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel new file mode 100644 index 00000000..39d92259 --- /dev/null +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "paged_attn.zig", + ], +) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md new file mode 100644 index 00000000..8ccc6fd9 --- /dev/null +++ b/zml/kernels/paged_attention/metal/README.md @@ -0,0 +1,28 @@ +# Metal paged attention + +This backend is automatically selected for Metal. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[The wrapper](metal.zig) requires split KV caches. It merges query head groups, +transposes queries and caches into the kernels' layouts, launches the +[Metal DSL kernels](paged_attn.zig), and restores the query's tagged layout for +the output. + +Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` +decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which +is `bf16` only: a threadgroup holds up to four query heads of a KV head, times +splits of the KV positions, at most 256 threads. Rows split over the positions +are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count +and the number of rows. Everything else, `f16` decode included, runs +`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of +8, 16 or 32 positions), then `zero_padding_rows`. + +Causal and non-causal standard masks, left sliding windows, and a custom scale +are passed to the kernels. Explicit key indices are rejected, and +`opts.sink` is currently ignored. Fused caches are rejected by +ordinary paged attention. + +When sharding heads, keep complete GQA groups together with matching query and +cache head ownership. Each local computation needs the full required key range, +features, and matching sequence metadata. diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig new file mode 100644 index 00000000..2cec68f5 --- /dev/null +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -0,0 +1,240 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const paged_attn = @import("paged_attn.zig"); + +const log = std.log.scoped(.@"zml/attention/metal"); + +test { + _ = paged_attn; +} + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); + + const num_kv_heads = q.dim(.hkv); + + const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); + const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + + const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; + + const out = dispatch(qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len, .{ + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + .scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + }); + + // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) + return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); +} + +const DispatchOptions = struct { + is_causal: bool, + /// Below 0 for global attention. + sliding_window: i32, + scale: f32, +}; + +/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled +/// otherwise. +/// `q` and the result are `[tokens, heads, head_dim]`, the caches +/// `[pages, block_size, kv_heads, head_dim]`, `block_table` +/// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` +/// `[seqs + 1]`, all i32. +fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table: zml.Tensor, seq_lens: zml.Tensor, query_start_len: zml.Tensor, opts: DispatchOptions) zml.Tensor { + const dtype = q.dtype(); + if (dtype != .bf16 and dtype != .f16) std.debug.panic("paged attention: Metal takes bf16 or f16 queries, not {t}", .{dtype}); + if (k_cache.dtype() != dtype or v_cache.dtype() != dtype) std.debug.panic("paged attention: Metal takes caches of the queries' {t}, not {t} and {t}", .{ dtype, k_cache.dtype(), v_cache.dtype() }); + + const total_q_tokens: i32 = @intCast(q.dim(0)); + const num_heads: i32 = @intCast(q.dim(1)); + const head_dim: i32 = @intCast(q.dim(2)); + const block_size: i32 = @intCast(k_cache.dim(1)); + const num_kv_heads: i32 = @intCast(k_cache.dim(2)); + const num_seqs: i32 = @intCast(block_table.dim(0)); + const max_blocks: i32 = @intCast(block_table.dim(1)); + + const vec_decode = opts.is_causal and total_q_tokens == num_seqs and + (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and + dtype == .bf16; + + if (vec_decode) { + const args: paged_attn.FaVecPaged.HostArgs = .{ + .num_heads = num_heads, + .n_kv = num_kv_heads, + .max_blocks = max_blocks, + .num_seqs = num_seqs, + .scale = opts.scale, + .sliding_window = opts.sliding_window, + }; + + const inputs: paged_attn.FaVecPaged.Kernel.Inputs = .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .seq_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }; + + const kv_capacity = max_blocks * block_size; + const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); + const nrows = total_q_tokens * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); + + const cfg: paged_attn.FaVecPaged.Cfg = .{ + .head_dim = head_dim, + .block_size = block_size, + .position_stride = num_kv_heads * head_dim, + .heads = launch.heads, + .splits = launch.splits, + }; + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + } + + const tile = paged_attn.PagedAttentionTiled.tileConfig(head_dim) orelse std.debug.panic("paged attention: head_dim {d} unsupported", .{head_dim}); + + const cfg: paged_attn.PagedAttentionTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(dtype), + .head_size = head_dim, + .block_size = block_size, + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + .causal = opts.is_causal, + }; + + const total_q_blocks = @divTrunc(total_q_tokens, tile[0]) + num_seqs; + + const out = paged_attn.PagedAttentionTiled.Kernel.call(.{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .context_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(num_heads), @intCast(total_q_blocks), 1 }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = paged_attn.PagedAttentionTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(num_kv_heads), + .scale = .of(opts.scale), + .softcapping = .of(@as(f32, 0)), + .max_num_blocks_per_seq = .of(max_blocks), + .q_stride = .of(num_heads * head_dim), + .kv_block_stride = .of(block_size * num_kv_heads * head_dim), + .kv_head_stride = .of(head_dim), + .num_seqs = .of(num_seqs), + .sliding_window = .of(opts.sliding_window), + }, + }).out; + + // The tiled kernel leaves the padding rows untouched, so they are zeroed in + // its output. Zeroing a fresh buffer first, from cu_seqlens_q alone, would + // make XLA share that buffer across every call and copy it for each. + return paged_attn.ZeroPaddingRows.Kernel.call(.{ .cu_seqlens_q = query_start_len, .out = out }, .{ .out = q.shape() }, .{ + .cfg = .{ .dtype = cfg.dtype }, + .grid = .{ @intCast(total_q_tokens), 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig new file mode 100644 index 00000000..04f4c1ac --- /dev/null +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -0,0 +1,908 @@ +//! Metal paged attention kernels: fa_vec_paged and its split-K reduce for +//! decode, and paged_attention_tiled with zero_padding_rows otherwise. + +const std = @import("std"); + +const metal = @import("../../../kernel/kernel.zig").metal; + +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVecPaged = struct { + pub const Cfg = struct { + /// The key and value head dimension. + head_dim: i32, + /// The page size. + block_size: i32, + /// The in-page position stride, n_kv * head_dim elements. + position_stride: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "max_blocks", .type = .int }, + .{ .name = "num_seqs", .type = .int }, + .{ .name = "scale", .type = .float }, + .{ .name = "sliding_window", .type = .int }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + max_blocks: i32, + num_seqs: i32, + scale: f32, + /// Below 0 for global attention. + sliding_window: i32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "seq_lens", "cu_seqlens_q" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k_cache = char_in, + .v_cache = char_in, + .block_tables = int_in, + .seq_lens = int_in, + .cu_seqlens_q = int_in, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const BS = cfg.block_size; + const PS = cfg.position_stride; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const group = num_heads.div(a.args.field("n_kv")); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + // Padding rows: FaVecPagedReduce zeroes them when there are splits. + var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + { + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); + var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); + { + dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); + loop.close(); + } + } + b.return_(); + padding.close(); + } + + const seq = sequenceOf(b, a, r); + const kv_len = seq.kv_len; + const kv_lo = seq.kv_lo; + const ic0_lo = kv_lo.div(C); + const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); + const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, ic0_lo.add(iwg)); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); + const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first) |*scope| scope.close(); + } + + /// The address of position `p`'s bf16 row in `cache`, through the page + /// table `pt`. + fn pagedRow(b: *Builder, cache: Value, pt: Value, p: Value, BS: i32, PS: i32) Value { + const page = pt.load(p.div(BS)).to(.uint).to(.ulong).mul(b.constant(.uint, BS * PS)); + const row = p.rem(BS).to(.uint).mul(b.constant(.uint, PS)); + return cache.add(page.add(row).mul(2)); + } +}; + +/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). +fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(idx)); + const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); + const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; +} + +fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { + const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); + return b.min(splits, chunks); +} + +/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. +pub const FaVecPagedReduce = struct { + pub const Cfg = struct { + splits: i32, + /// The head dimension. + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged_reduce", + .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVecPaged.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .seq_lens = int_in, + .cu_seqlens_q = int_in, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + + const live = b.local(.int, 0); + var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + { + live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); + real.close(); + } + var empty = b.openIf(live.le(0)); + { + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); + lane0.close(); + } + b.return_(); + empty.close(); + } + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// zero_padding_rows: zeroes the output rows past the last real token. +pub const ZeroPaddingRows = struct { + pub const Cfg = struct { dtype: Type }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "zero_padding_rows", + .inputs = &.{ "cu_seqlens_q", "out" }, + .outputs = &.{"out"}, + .constants = &.{ "num_seqs", "row_stride" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.dtype.scalar) { + .bfloat, .half => {}, + else => std.debug.panic("zero_padding_rows: {s} unsupported (half or bfloat)", .{cfg.dtype.name()}), + } + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = cfg.dtype } }, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = .{ .constant = .int }, + .row_stride = .{ .constant = .int }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .ntg = .{ .threads_per_threadgroup = .uint3 }, + }); + const row = a.tgp.x().to(.int); + b.returnIf(row.lt(a.cu_seqlens_q.load(a.num_seqs))); + const dst = a.out.add(row.to(.ulong).mul(a.row_stride.to(.ulong))); + var loop = b.openFor(a.tpt.x().to(.int), a.row_stride, a.ntg.x().to(.int)); + { + dst.store(loop.iv, b.constant(cfg.dtype, 0)); + loop.close(); + } + } +}; + +/// paged_attention_tiled: FlashAttention-2 over 8x8 simdgroup matrices, causal +/// or bidirectional, one threadgroup per (query head, query block). +pub const PagedAttentionTiled = struct { + pub const Cfg = struct { + /// half or bfloat. + dtype: Type, + head_size: i32, + block_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + /// Bidirectional when false. + causal: bool = true, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "paged_attention_tiled", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "context_lens", "cu_seqlens_q" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "softcapping", "max_num_blocks_per_seq", "q_stride", "kv_block_stride", "kv_head_stride", "num_seqs", "sliding_window" }, + .run = run, + }); + + /// What the MSL instantiates: half or bfloat, a head size of tileConfig + /// with its tiles, and pages of 8, 16 or 32 positions. + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + (cfg.block_size == 8 or cfg.block_size == 16 or cfg.block_size == 32) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + /// The (bq, tile_kv, num_threads) of a head size, null when unsupported. + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + /// Reduces `v` with `op` across the lanes that hold one row of an 8x8 + /// simdgroup_matrix fragment. + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + /// Sets lane `lane` of the vector `p[i]` to `x`. + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("paged_attention_tiled: {s} at head size {d}, block size {d} (8, 16 or 32) and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.block_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BLOCK_SIZE = cfg.block_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const float_arg: ArgSpec = .{ .constant = .float }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .v_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .num_kv_heads = int_arg, + .scale = float_arg, + .softcapping = float_arg, + .block_tables = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .context_lens = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .max_num_blocks_per_seq = int_arg, + .q_stride = int_arg, + .kv_block_stride = int_arg, + .kv_head_stride = int_arg, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = int_arg, + .sliding_window = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("paged_attention_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_block_global_idx = a.tgp.y().to(.int); + const num_heads = a.tgpg.x().to(.int); + const num_queries_per_kv = num_heads.div(a.num_kv_heads); + const kv_head_idx = head_idx.div(num_queries_per_kv); + const kv_token_stride = a.num_kv_heads.mul(a.kv_head_stride); + + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.num_seqs); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).div(BQ).add(mid).le(q_block_global_idx), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + // The grid is an upper bound on the query blocks: the extra ones land past + // the last sequence and clamp to it. + const seq_idx = b.min(first_seq.rvalue(), a.num_seqs.sub(1)); + + const q_seq_start = a.cu_seqlens_q.load(seq_idx); + const cur_batch_query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(q_seq_start); + const q_block_start = q_seq_start.div(BQ).add(seq_idx); + const q_block_local = q_block_global_idx.sub(q_block_start); + const q_pos_start = q_block_local.mul(BQ); + + b.returnIf(q_pos_start.ge(cur_batch_query_len)); + + const seq_len = a.context_lens.load(seq_idx).to(.int); + const context_len = seq_len.sub(cur_batch_query_len); + const valid_q = b.min(BQ, cur_batch_query_len.sub(q_pos_start)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(a.q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const block_table = a.block_tables.add(seq_idx.mul(a.max_num_blocks_per_seq)); + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + if (cfg.causal) { + var past = b.openIf(tile_start.gt(context_len.add(q_pos_start).add(valid_q).sub(1))); + { + tiles.breakLoop(); + past.close(); + } + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const pb = block_table.load(kv_pos.div(BLOCK_SIZE)).to(.long); + const off = pb.mul(a.kv_block_stride).add(kv_pos.rem(BLOCK_SIZE).mul(kv_token_stride)).add(kv_head_idx.mul(a.kv_head_stride)); + const k_ptr = a.k_cache.add(off); + const v_ptr = a.v_cache.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + const k = k_ptr.add(d).to(LoadUnit).load(0); + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k); + const v = v_ptr.add(d).to(LoadUnit).load(0); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const min_q_abs_pos = context_len.add(q_pos_start); + var tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(a.softcapping.le(0.0)).bitAnd(a.sliding_window.lt(0)); + if (cfg.causal) tile_no_mask = tile_no_mask.bitAnd(tile_start.add(TILE_KV).sub(1).lt(min_q_abs_pos)); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = b.local(.float, s_reg.load(k).at(jj).mul(scale_log2)); + var capped = b.openIf(a.softcapping.gt(0.0)); + { + const s_orig = s.div(log2e); + s.assign(a.softcapping.mul(b.precise().tanh(s_orig.div(a.softcapping))).mul(log2e)); + capped.close(); + } + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + var masked = row_masked.bitOr(kv_pos.ge(seq_len)); + if (cfg.causal) masked = masked.bitOr(kv_pos.gt(q_abs_pos)); + const before_window = a.sliding_window.ge(0).bitAnd(kv_pos.lt(q_abs_pos.add(1).sub(a.sliding_window))); + masked = masked.bitOr(before_window); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const out_base = a.out.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + const total = valid_q.mul(HEAD_SIZE); + var write_out = b.openFor(thread_idx, total, NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = O_smem.load(r.mul(LD).add(d)).to(T); + out_base.store(r.mul(a.q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the paged attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = PagedAttentionTiled.tileConfig(head_size).?; + for ([_]bool{ true, false }) |causal| { + allocator.free(try PagedAttentionTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .block_size = 16, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2], .causal = causal })); + } + } + } +} diff --git a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..6a0e4e87 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "mosaic_tpu.zig", + "ragged_attention.zig", + ], +) diff --git a/zml/kernels/paged_attention/mosaic_tpu/README.md b/zml/kernels/paged_attention/mosaic_tpu/README.md new file mode 100644 index 00000000..6fff585c --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/README.md @@ -0,0 +1,28 @@ +# Mosaic TPU paged attention + +This backend is automatically selected for TPU. See the +[parent README](../README.md) for the attention equation and packed sequence +metadata contract. + +[The wrapper](mosaic_tpu.zig) requires fused `KvCache.dense` storage with shape +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal the logical query head +dimension rounded up to a multiple of 128, with unused cache features zeroed. +Queries are padded before execution and outputs trimmed afterward. The default +scale uses the logical head dimension. Accepted query/cache dtypes are `bf16`, +`f16`, and `f32`. + +Causal and non-causal standard masks and left sliding windows are supported. +Explicit key indices are rejected. `opts.sink` is currently ignored. Split +caches are rejected by ordinary paged attention, and the generic +`KvCache.update` method does not implement fused-cache writes. + +The wrapper partitions heads on the model mesh and replicates sequence metadata. +It counts sequences with nonempty query intervals and passes that count to the +ragged kernel, so active sequences must precede trailing inactive entries. +Each local head computation must have the full required key range and features. + +The shared split-cache comparison test does not exercise this backend. Wrapper +unit tests cover head-dimension alignment and configuration, including scale +and standard mask options; they do not establish runtime support for every +shape or dtype. diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..84716379 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,275 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ragged_paged = @import("platforms/tpu/ragged_paged"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const ragged_attention = @import("ragged_attention.zig"); + +const log = std.log.scoped(.tpu_attention); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(ragged_attention); +} + +pub const mosaic_tpu = struct { + const PreparedInputs = struct { + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + }; + + pub fn alignedKernelHeadDim(head_dim: usize) usize { + stdx.debug.assert(head_dim > 0, "mosaic_tpu ragged paged attention head_dim must be non-zero", .{}); + const alignment: usize = 128; + return (std.math.divCeil(usize, head_dim, alignment) catch unreachable) * alignment; + } + + pub const Options = struct { + is_prefill: bool, + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + pages_per_compute_block: usize = 1, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + opts: Options, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + + pub fn init(opts: Options) Parameters { + return .{ + .opts = opts, + .block_table = zml.Tensor.init(.{ .b = opts.batch_size, .p = opts.max_num_pages }, .i32), + .seq_lens = zml.Tensor.init(.{ .b = opts.batch_size }, .i32), + .query_start_len = zml.Tensor.init(.{ .b = opts.batch_size + 1 }, .i32), + }; + } + + pub fn options(self: Parameters) Options { + return self.opts; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + return allocation_size; + } + }; + + fn raggedPagedKernelCall( + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + cfg: ragged_paged.Cfg, + ) zml.Tensor { + const mlir_ctx = zml.Compiler.current().mlir_ctx; + const out = ragged_attention.Kernel.call( + .{ + .kv_lens = seq_lens, + .page_indices = block_table, + .cu_q_lens = query_start_len, + .seq_buf_idx = .zeroes(.init(.{2}, .i32)), + .num_seqs = num_seqs, + .q = q, + .kv_pages = kv_pages, + }, + .{ .o = q.shape() }, + .{ + .cfg = cfg, + .extras = .{ + .vmem_limit_bytes = cfg.vmem_limit_bytes, + .additional_attributes = &.{ + .named(mlir_ctx, "kernel_name", .string(mlir_ctx, "ragged_paged_attention_kernel")), + .named(mlir_ctx, "mhlo.frontend_attributes", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "kernel_metadata", .string(mlir_ctx, "{}")), + })), + }, + }, + }, + ); + return out.o; + } + + fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { + const start = query_start_len.slice(.b, .{ .end = query_start_len.dim(.b) - 1 }); + const end = query_start_len.slice(.b, .{ .start = 1 }); + const query_lens = end.sub(start); + return query_lens + .cmp(.GT, .zeroes(query_lens.shape())) + .convert(.i32) + .sum(.b) + .reshape(.{1}); + } + + fn cfgDtype(dtype: zml.DataType) @FieldType(ragged_paged.Cfg, "q_dtype") { + const cast = zml.kernel.mosaic_tpu.from(dtype); + return switch (cast) { + .bf16, .f16, .f32 => cast, + else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16/f16/f32, got {}", .{dtype}), + }; + } + + inline fn restoreQueryHeads(q_template: zml.Tensor, q_out: zml.Tensor) zml.Tensor { + const restored = q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); + if (restored.dim(.hd) == q_template.dim(.hd)) return restored; + stdx.debug.assert( + restored.dim(.hd) > q_template.dim(.hd), + "mosaic_tpu ragged paged attention cannot restore output from head_dim {} to {}", + .{ restored.dim(.hd), q_template.dim(.hd) }, + ); + return restored.slice(.hd, .{ .end = q_template.dim(.hd) }); + } + + inline fn alignQueryHeadDimForKernel(q: zml.Tensor, target_head_dim: i64) zml.Tensor { + if (q.dim(.hd) == target_head_dim) return q; + stdx.debug.assert( + q.dim(.hd) < target_head_dim, + "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", + .{ q.dim(.hd), target_head_dim }, + ); + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); + } + + inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); + const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); + stdx.debug.assert( + cache_head_dim == alignedKernelHeadDim(logical_head_dim), + "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", + .{ cache_head_dim, logical_head_dim }, + ); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); + const query_start_len = parameters.query_start_len; + + return .{ + .q = q_ragged, + .kv_pages = kv_pages, + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, + .query_start_len = query_start_len, + .num_seqs = activeSequenceCount(query_start_len), + }; + } + + fn buildCfg(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) ragged_paged.Cfg { + const q_shape = prepared_inputs.q.shape(); + const kv_pages_shape = prepared_inputs.kv_pages.shape(); + const block_table_shape = prepared_inputs.block_table.shape(); + + const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); + const max_num_seqs = block_table_shape.dim(.b); + const num_q_heads = q_shape.dim(.h); + stdx.debug.assert( + q_shape.dim(.hd) == kv_pages_shape.dim(.hd), + "mosaic_tpu ragged paged attention expects q and KV cache head_dim to match, got {} and {}", + .{ q_shape.dim(.hd), kv_pages_shape.dim(.hd) }, + ); + stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); + const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch + stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); + const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); + const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); + + const sliding_window: ?i64 = if (opts.mask.slidingWindow()) |window| @intCast(window) else null; + + return .{ + .num_q_tokens = q_token_count, + .num_q_heads = num_q_heads, + .num_kv_heads = num_kv_heads, + .head_dim = q_shape.dim(.hd), + .total_num_pages = kv_pages_shape.dim(.page), + .page_size = kv_pages_shape.dim(.k_chunk), + .max_num_seqs = max_num_seqs, + .pages_per_seq = @intCast(parameters.opts.max_num_pages), + .num_kv_pages_per_block = @intCast(parameters.opts.pages_per_compute_block), + .num_queries_per_block = num_queries_per_block, + .q_dtype = cfgDtype(q_shape.dtype()), + .kv_dtype = cfgDtype(kv_pages_shape.dtype()), + .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + }; + } + + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); + const prepared = prepareInputs(parameters, q, kv_cache); + const cfg = buildCfg(prepared, parameters, opts); + const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); + const restored = restoreQueryHeads(q, q_out); + stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); + return restored; + } +}; + +test "mosaic_tpu aligns kernel head dimensions to 128" { + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(64)); + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(128)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(180)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(256)); +} + +test "mosaic_tpu cfg uses kernel head dim and logical scale" { + const parameters = mosaic_tpu.Parameters.init(.{ + .is_prefill = false, + .batch_size = 1, + .max_num_pages = 2, + .max_seqlen_k = 32, + .max_token_count = 1, + .num_heads = 8, + .num_kv_heads = 2, + .head_dim = 64, + }); + const prepared_inputs: mosaic_tpu.PreparedInputs = .{ + .q = zml.Tensor.init(.{ .b = 1, .h = 8, .hd = 128 }, .bf16), + .kv_pages = zml.Tensor.init(.{ .page = 2, .k_chunk = 16, .hkv = 4, .hd = 128 }, .bf16), + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, + .query_start_len = parameters.query_start_len, + .num_seqs = zml.Tensor.init(.{1}, .i32), + }; + + const cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{}); + try std.testing.expectEqual(@as(i64, 128), cfg.head_dim); + try std.testing.expect(cfg.is_causal); + try std.testing.expectApproxEqAbs( + @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), + cfg.sm_scale, + 0.000001, + ); + + const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .causal = false } } }); + try std.testing.expect(!non_causal_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, null), non_causal_cfg.sliding_window); + + const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .sliding_window = 16 } } }); + try std.testing.expect(window_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, 16), window_cfg.sliding_window); +} diff --git a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig new file mode 100644 index 00000000..30536306 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig @@ -0,0 +1,14 @@ +const ragged_paged = @import("platforms/tpu/ragged_paged"); + +const zml = @import("../../../zml.zig"); + +test { + _ = Kernel; +} + +pub const Kernel = zml.kernel.mosaic_tpu.Kernel(ragged_paged.Cfg, .{ + .name = "ragged_paged_attention_kernel", + .inputs = &.{ "kv_lens", "page_indices", "cu_q_lens", "seq_buf_idx", "num_seqs", "q", "kv_pages" }, + .outputs = &.{"o"}, + .run = ragged_paged.run, +}); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig new file mode 100644 index 00000000..4a97a38e --- /dev/null +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -0,0 +1,502 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../zml.zig"); +pub const AttentionMask = @import("options.zig").AttentionMask; +pub const AttentionOptions = @import("options.zig").AttentionOptions; +pub const cute = @import("cute/cute.zig"); +const metal = @import("metal/metal.zig"); +const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const triton = @import("triton/triton.zig"); + +const PagedAttention = @This(); + +test { + _ = tcl; + std.testing.refAllDecls(Backend); + std.testing.refAllDecls(Options); + std.testing.refAllDecls(Parameters); + std.testing.refAllDecls(KvCache); +} + +pub const Backend = enum { + cute, + triton, + mosaic_tpu, + metal, + stablehlo, + tcl, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => if (cute.isAvailable(platform)) .cute else .triton, + .rocm => .triton, + .oneapi => .triton, + .tpu => .mosaic_tpu, + .metal => .metal, + .cpu => .stablehlo, + .furiosa => .tcl, + .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .cute => cute.isAvailable(platform), + .stablehlo => true, + .tcl => platform.target == .furiosa, + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + .cpu, .tpu, .neuron, .metal, .furiosa => false, + }, + .metal => platform.target == .metal, + .mosaic_tpu => platform.target == .tpu, + }; + } +}; + +pub const Options = union(Backend) { + cute: cute.Options, + triton: triton.paged.Options, + mosaic_tpu: tpu.mosaic_tpu.Options, + metal: metal.Options, + stablehlo: triton.paged.Options, + tcl: tcl.paged.Options, + + const Args = struct { + backend: Backend, + is_prefill: bool, + batch_size: u32, + batch_size_prefill: ?u32 = null, + batch_size_decode: ?u32 = null, + seq_len: u32, + max_num_pages: u32, + max_token_count: u32, + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + max_seqlen_q: u32, + }; + + pub fn fromBackend(args: Args) Options { + return switch (args.backend) { + .mosaic_tpu => .{ + .mosaic_tpu = .{ + .is_prefill = args.is_prefill, + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, + }, + }, + inline .cute, .triton, .metal, .stablehlo, .tcl => |t| @unionInit(Options, @tagName(t), .{ + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }), + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + inline else => |v| v.maxNumPages(), + }; + } +}; + +pub const Parameters = union(Backend) { + cute: cute.Parameters, + triton: triton.paged.Parameters, + mosaic_tpu: tpu.mosaic_tpu.Parameters, + metal: metal.Parameters, + stablehlo: triton.paged.Parameters, + tcl: tcl.paged.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + zml.Tensor.onMemoryAll(&self, memory); + return self; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return zml.Tensor.toMemoryAll(self, memory); + } +}; + +pub const KvCache = union(enum) { + split: struct { + k: zml.Tensor, + v: zml.Tensor, + }, + // TODO: rename `dense` to `fused`. + dense: zml.Tensor, + + pub fn update( + self: KvCache, + new_k: zml.Tensor, + new_v: zml.Tensor, + slot_mapping: zml.Tensor, + chunk_size: u32, + backend: Backend, + ) KvCache { + const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); + + const kv: KvCache = switch (self) { + .split => |split| switch (backend) { + .cute, .triton, .mosaic_tpu, .metal, .stablehlo, .tcl => .{ + .split = .{ + .k = split.k.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.k), + .v = split.v.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_v, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.v), + }, + }, + }, + .dense => @panic("TODO"), + }; + + return kv; + } + + pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { + const page_index = slot_mapping.divByConst(page_size); + const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); + return .{ page_index, offset }; + } +}; + +pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + _ = k; + _ = v; + const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); +} + +const Attention = struct { + parameters: Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .cute => |p| switch (args.kv_cache) { + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (args.kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + }, + .metal => |metal_parameters| switch (args.kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .tcl => |params| switch (args.kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, + }; + } +}; + +test "Backend.auto selects mosaic_tpu on TPU" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .tpu, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.tpu), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); +} + +test "Backend.auto selects triton on oneAPI" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .oneapi, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.oneapi), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); +} + +test pagedAttention { + const platform = zml.testing.env(); + const reference_backend: Backend = if (Backend.triton.isAvailable(platform)) + .triton + else if (Backend.metal.isAvailable(platform)) + .stablehlo + else + return error.SkipZigTest; + const io = std.testing.io; + const allocator = std.testing.allocator; + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const num_pages = 64; + const page_size = 16; + const max_num_pages = 16; + const prefill_token_count = 32; + const num_prefill = 1; + const num_decode = 6; + const active_batch_size = num_prefill + num_decode; + const batch_size = active_batch_size + 1; + const query_token_count = prefill_token_count + num_decode; + const dt: zml.DataType = .bf16; + const tp = zml.testing.tpMesh(); + const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); + const partition = .{ .hkv = .model }; + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .kv_cache = .{ + .split = .{ + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + }, + }, + }; + + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); + defer rng_q.deinit(); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_k.deinit(); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_kv_cache.deinit(); + + const triton_options_args: Options.Args = .{ + .backend = .triton, + .is_prefill = true, + .batch_size = batch_size, + .batch_size_prefill = num_prefill, + .batch_size_decode = num_decode, + .seq_len = max_num_pages * page_size, + .max_num_pages = max_num_pages, + .max_token_count = 16 * page_size, + .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), + .num_kv_heads = @intCast(tensors.q.dim(.hkv)), + .head_dim = @intCast(tensors.q.dim(.hd)), + .max_seqlen_q = 16 * 2, + }; + const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); + var q = try rng_q.eval(allocator, io, {}); + defer q.deinit(); + var new_k = try rng_k.eval(allocator, io, {}); + defer new_k.deinit(); + var new_v = try rng_k.eval(allocator, io, {}); + defer new_v.deinit(); + + var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; + kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.k.deinit(); + kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.v.deinit(); + + const block_table: [batch_size][max_num_pages]i32 = .{ + // prefilling pages 9-10 + .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + // generation + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + }; + + const seq_lens: [batch_size]i32 = .{ + 4 * page_size, + 4 * page_size + 3, + 7 * page_size + 11, + 15 * page_size + 9, + 4 * page_size + 3, + 4 * page_size + 3, + 4 * page_size + 3, + 0, + }; + + const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; + var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); + + const all_backends = std.enums.values(Backend); + // stablehlo ignores these options, mosaic_tpu requires a dense + // KV cache instead of the split cache used by this test. + const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal, .tcl }; + const test_cases = [_]struct { + name: []const u8, + attention_options: AttentionOptions, + backends: []const Backend, + }{ + .{ + .name = "unbounded", + .attention_options = .{ .mask = .{ .standard = .{} } }, + .backends = all_backends, + }, + .{ + .name = "non_causal", + .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, + .backends = &option_sensitive_backends, + }, + .{ + .name = "sliding_window", + .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, + .backends = &option_sensitive_backends, + }, + }; + + for (test_cases) |test_case| { + var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); + defer { + for (std.enums.values(Backend)) |backend| { + if (results_per_backend.get(backend)) |slice| slice.free(allocator); + } + } + + for (test_case.backends) |backend| { + if (!backend.isAvailable(platform)) { + std.log.warn("paged_attention backend {t} not available", .{backend}); + continue; + } + std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); + + var backend_options_args = triton_options_args; + backend_options_args.backend = backend; + const parameters: Parameters = .init(.fromBackend(backend_options_args)); + + const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const partitioning: Partitioning = .{ + .in_specs = .{ + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), + .q = head_spec, + .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, + .opts = .{ .mask = .{ .standard = {} }, .sink = null }, + }, + .out_specs = head_spec, + }; + + const exe = try platform.compileFn( + allocator, + io, + pagedAttention, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, + ); + defer exe.deinit(); + + const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { + .triton => triton_parameters_d, + inline .cute, .metal, .mosaic_tpu, .stablehlo, .tcl => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + .block_table = triton_parameters_d.triton.block_table, + .seq_lens = triton_parameters_d.triton.seq_lens, + .query_start_len = triton_parameters_d.triton.query_start_len, + }), + }; + + var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{ .mask = .{ .standard = {} }, .sink = null } }); + defer output_d.deinit(); + results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); + } + + const reference = results_per_backend.get(reference_backend) orelse { + std.log.warn("test pagedAttention {s} has no {t} reference", .{ test_case.name, reference_backend }); + continue; + }; + + var num_failed: u32 = 0; + for (std.enums.values(Backend)) |backend| { + if (backend == reference_backend) continue; + const output_h = results_per_backend.get(backend) orelse continue; + + const tolerance: zml.testing.CompareOpts = .{ + .absolute_tolerance = 1e-2, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-6, + }; + + zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + num_failed += 1; + std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ + test_case.name, + backend, + reference_backend, + }); + }, + else => |e| return e, + }; + } + + if (num_failed > 0) return error.TestUnexpectedResult; + } +} diff --git a/zml/kernels/paged_attention/stablehlo/BUILD.bazel b/zml/kernels/paged_attention/stablehlo/BUILD.bazel new file mode 100644 index 00000000..f60464f4 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = ["stablehlo.zig"], +) diff --git a/zml/kernels/paged_attention/stablehlo/README.md b/zml/kernels/paged_attention/stablehlo/README.md new file mode 100644 index 00000000..34144345 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/README.md @@ -0,0 +1,30 @@ +# StableHLO paged attention + +This fallback is automatically selected for CPU and Furiosa. See the +[parent README](../README.md) for the general attention equation, packed +sequence metadata, and page addressing contract. + +[The implementation](stablehlo.zig) iterates over sequences, query chunks, and +KV pages using StableHLO control flow. It merges partial softmax states with +`f32` maxima, exponential sums, and weighted-value accumulators, then converts +the final output to the query dtype. Query-key dot products use the input dtype. + +## Current limitations + +- Only split KV caches work. Although public dispatch forwards `.dense`, this + implementation accesses `.split`. Latent caches are rejected by dispatch. +- Attention is always causal: `causal = false` is currently ignored. Sliding + windows and explicit key indices are rejected. `opts.sink` is ignored; custom + scale is supported. +- Multi-query chunks are processed a full page at a time, and the packed output + cursor advances by the page size. Use page-multiple query chunks; arbitrary + partial query chunks are not covered by this implementation. Single-query + decode uses a separate one-row path. +- The sequence loop stops at the first empty query interval. Active sequences + must precede trailing inactive entries. + +The shared tests in [paged_attention.zig](../paged_attention.zig) use this path +as the causal comparison reference with `bf16`, page size 16, mixed +prefill/decode, shared prefix pages, partial final KV pages, and trailing +inactive sequences. Those cases do not establish support for every shape or +dtype, or conformance for the unsupported options above. diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig new file mode 100644 index 00000000..55ee01d5 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -0,0 +1,262 @@ +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const triton = @import("../triton/triton.zig"); + +pub fn pagedAttention( + parameters: triton.paged.Parameters, + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + opts: AttentionOptions, +) zml.Tensor { + switch (opts.mask) { + .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), + .indices => @panic("Sparse attention mask is not supported on StableHLO"), + } + + const page_size = k.dim(.k_chunk); + const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); + + const final_state = zml.ops.@"while"( + AttentionLoop, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, + .{ + // i32 to match triton conventions + .slot_id = .scalar(0, .i32), + .seq_id = .scalar(0, .i32), + .q_page_idx = .scalar(0, .i32), + .k_page_idx = .scalar(0, .i32), + // Note: we do all partial softmax in f32 + .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), + .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), + .out = .zeroes(padded_q.shape()), + }, + ); + + return final_state.out.slice(.b, .{ .end = q.dim(.b) }); +} + +const AttentionLoop = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: triton.paged.Parameters, + opts: AttentionOptions, + + const While = @This(); + + pub const State = struct { + slot_id: zml.Tensor, + seq_id: zml.Tensor, + q_page_idx: zml.Tensor, + k_page_idx: zml.Tensor, + partial_softmax_prefill: PartialSoftmax, + partial_softmax_decode: PartialSoftmax, + out: zml.Tensor, + }; + + pub fn cond(self: While, state: State) zml.Tensor { + const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); + const has_queries = query_start.cmp(.LT, query_end).asScalar(); + return has_more_sequences.logical(.AND, has_queries); + } + + pub fn body(self: While, state_: State) State { + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); + const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); + + return zml.ops.@"if"( + struct { + const IfPrefill = @This(); + while_body: While, + state: State, + seq_num_queries: zml.Tensor, + page_size: u32, + + /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. + pub fn onTrue(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = page_size; + + // seq_lens includes the current queries; subtracting their count gives the context length. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); + const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + // The current K page is processed in this iteration. Continue if the next page contains + // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + } + + /// Decode version: num_slots == 1, we multiply one query with a page of keys. + pub fn onFalse(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = 1; + + // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); + const q_offset = seq_len.sub(if_ctx.seq_num_queries); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + } + }, + .{ + .while_body = self, + .state = state_, + .seq_num_queries = seq_num_queries_, + .page_size = page_size_, + }, + .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), + ); + } + + pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { + const k_offset = state.k_page_idx.scale(page_size); + const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); + + const active_k_page_id = self.parameters.block_table.slices( + .{ .b, .p }, + &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, + ); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); + + const dtype = self.q.dtype(); + const attn_mask: zml.Tensor = attn_mask: { + const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); + + const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); + const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); + + const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); + const minus_inf = zml.Tensor.constant(dtype.minValue()); + break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); + }; + + const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); + var attn_weights = active_q.dot(active_k, .hd).scale(scale); + attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); + + const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; + // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. + const precise_dt = og_softmax.values.dtype(); + var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); + partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); + partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); + partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); + + return og_softmax.merge(partial); + } + + fn nextQueryStart(self: While, state: State) zml.Tensor { + return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { + return zml.ops.if2( + query_has_more_keys, + // If the query has more pages: update partial softmax with attention on current page, delay updating output. + State{ + .slot_id = state.slot_id, + .seq_id = state.seq_id, + .q_page_idx = state.q_page_idx, + .k_page_idx = state.k_page_idx.addConstant(1), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, + .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, + .out = state.out, + }, + // Query has no more pages, we commit softmax to output and increment slot + State{ + .k_page_idx = .scalar(0, .i32), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), + .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, + .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), + .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), + // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. + .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), + // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. + .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), + }, + ); + } +}; + +pub const PartialSoftmax = struct { + values: zml.Tensor, + exp_sum: zml.Tensor, + max_value: zml.Tensor, + + pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { + return .{ + .values = .zeroes(q_shape), + .exp_sum = .zeroes(q_shape.drop(.hd)), + .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), + }; + } + + pub fn reset(self: PartialSoftmax) PartialSoftmax { + return .zeroes(self.values.shape()); + } + + pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { + // Rescale self and other using the new global_max. + const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); + const new_self = self.rescale(global_max); + const new_other = other.rescale(global_max); + + // Now that self and other are using the same scale, we can just add them: + return .{ + .max_value = global_max, + .values = new_self.values.add(new_other.values), + .exp_sum = new_self.exp_sum.add(new_other.exp_sum), + }; + } + + /// Update max_value and rescale attn and exp_sum accordingly. + pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { + const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); + const sum_dtype = self.exp_sum.dtype(); + return .{ + .max_value = max_value, + .values = self.values.mul(max_diff_exp.broad(self.values.shape())), + .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), + }; + } + + /// Divides the intermediary results by the exp_sum to get the proper attention values. + pub fn finalize(self: PartialSoftmax) zml.Tensor { + return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); + } +}; + +/// Compute softmax over a chunk. +/// Returns intermediary results to allow aggregating later. +pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { + const a = self.axis(axis); + const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); + const out = self.sub(max_val.broad(self.shape())).exp(); + return .{ + .values = out, + .exp_sum = out.convert(.f32).sum(a).squeeze(a), + .max_value = max_val, + }; +} diff --git a/zml/kernels/paged_attention/tcl/BUILD.bazel b/zml/kernels/paged_attention/tcl/BUILD.bazel new file mode 100644 index 00000000..4057eea3 --- /dev/null +++ b/zml/kernels/paged_attention/tcl/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = ["tcl.zig"], +) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig new file mode 100644 index 00000000..d04b684e --- /dev/null +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -0,0 +1,1781 @@ +//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. +//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. +//! `Repro:` files are in the reproducer repository, platform/furiosa. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; +const AttentionOptions = @import("../options.zig").AttentionOptions; +const triton = @import("../triton/triton.zig"); + +/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to +/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. +/// Remove once 8192-key chunks compile. +const chunk_keys = 4096; +const flash_chunk_pairs = 512 * 2048; +const prefill_chunk_keys = 2048; +const prefill_block = 32; + +/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout +/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. +/// Remove once any count compiles. +fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { + const blocks = @divTrunc(tokens + sequences * (block - 1), block); + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); +} + +fn prefillBlock(tokens: i64, sequences: i64) i64 { + const short = prefill_block / 2; + return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; +} + +/// A straight-line head over each token's first `head` keys, then a device loop over +/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. +pub const Plan = struct { + head: i64, + chunk: i64, + group: i64, +}; + +pub var calibration_plan: ?Plan = null; + +pub const paged = struct { + pub const Options = triton.paged.Options; + pub const Parameters = triton.paged.Parameters; + + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. + /// Sinks are one logit per query head. + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); + stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); + stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); + stdx.debug.assert(q.dim(.hkv) == k_cache.dim(.hkv) and q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hg) > 0, "TCL attention query and cache head dimensions must match", .{}); + stdx.debug.assert(parameters.block_table.dtype() == .i32 and parameters.seq_lens.dtype() == .i32 and parameters.query_start_len.dtype() == .i32, "TCL attention metadata must be I32", .{}); + stdx.debug.assert(parameters.block_table.dim(.b) == parameters.seq_lens.dim(.b) and parameters.query_start_len.dim(.b) == parameters.seq_lens.dim(.b) + 1, "TCL attention metadata batch dimensions must match", .{}); + stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); + stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); + const compiler = zml.Compiler.current(); + stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); + const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch + std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); + defer attributes.deinit(); + const first_pe = attributes.get("pe_start"); + const last_pe = attributes.get("pe_end"); + stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and + last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + return attend(parameters, q, k_cache, v_cache, opts); + } + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); + const page = k_cache.dim(.k_chunk); + // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to + // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. + // Remove once 16- to 48-key decode compiles. + const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; + // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 + // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: + // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. + const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; + const alignment = if (key_alignment <= chunk_keys) key_alignment else page; + const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); + const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); + const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); + const rows = block_count * block; + // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. + const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; + const chunk = @min(keys, chunk_limit); + var cfg: Cfg = .{ + .tokens = q.dim(.b), + .sequences = parameters.seq_lens.dim(.b), + .b = block_count, + .i = block, + .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, + .chunk = chunk, + .hkv = q.dim(.hkv), + .g = q.dim(.hg), + .d = q.dim(.hd), + .pages = k_cache.dim(.page), + .pg = page, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .decode = decode, + .use_sink = opts.sink != null, + .is_causal = opts.mask.isCausal(), + .window = @intCast(opts.mask.slidingWindow() orelse 0), + }; + const plain_decode = decode and !cfg.use_sink and cfg.window == 0; + // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens + // over a non-power-of-two page count fail to compile, so straight decode reads + // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, + // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. + const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; + const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; + const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); + if (plan.head > 0 and plan.chunk > 0) { + stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); + return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); + } + if (plan.chunk == 0 and straight) { + return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ + .t = cfg.tokens, + .a = straight_keys, + .group = plan.group, + .hkv = cfg.hkv, + .g = cfg.g, + .d = cfg.d, + .pages = cfg.pages, + .pg = page, + .scale = cfg.scale, + .is_causal = cfg.is_causal, + .window = cfg.window, + .use_sink = cfg.use_sink, + }).convert(q.dtype()); + } + cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; + cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size + // other than a power of two fails ("Inconsistent tensor group + // offsets"): pad the groups with zero queries, whose outputs are + // dropped. Repro: + // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, + // platform/furiosa). Remove once any group size compiles. + const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); + const extra = groups - cfg.g; + cfg.g = groups; + const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); + const inputs: OneShot.Inputs = .{ + .q = queries, + .pages = table, + .seq_lens = parameters.seq_lens, + .query_offsets = parameters.query_start_len, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), + }; + const outputs: OneShot.Outputs = .{ .out = queries.shape() }; + const out = if (cfg.a == cfg.chunk) + OneShot.call(inputs, outputs, .{ .cfg = cfg }).out + else + Flash.call(inputs, outputs, .{ .cfg = cfg }).out; + return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); + } +}; + +fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { + // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in + // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. + // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. + if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { + if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; + } + if (straight and cfg.window > 0) { + const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); + const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; + if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) + return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; + } + if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) + return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; + if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; +} + +const window_chunk_keys = 1024; +const window_flash_chunks = 16; +const batch_window_flash_chunks = 4; + +/// Widest block table the straight-line decode kernel takes for one token. +/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot +/// propagate gather bounds outside a loop over a token axis. Repro: +/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. +const straight_decode_keys = 32768; + +/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC +/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. +/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. +const decode_keys = 65536; + +const chunked_decode_tokens = 16; +const straight_batch_decode_keys = 8192; + +/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero +/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more +/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. +const unmasked_decode_keys = 1 << 17; + +const masked_decode_keys = 4096; + +const DecodeCfg = struct { + t: i64, + a: i64, + group: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + is_causal: bool, + window: i64, + use_sink: bool, +}; + +/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { + const columns = @divExact(cfg.a, cfg.pg); + const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); + const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .pages = zml.Shape.init(.{ .t = cfg.t, .p = columns }, .i32), + .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), + .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), + .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); + return Decode.call(.{ + .q = q.convert(.bf16), + .pages = meta.pages, + .bounds = meta.bounds, + .mask = meta.mask, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + .sink = sink, + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; + +/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC +/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, +/// repro_row_broadcast_past_4096.py. Remove once both compile. +const DecodeMeta = tcl.Kernel(MetaCfg, .{ + .name = "paged_decode_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "pages", "bounds", "mask", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: MetaCfg) tcl.FinishError!void { + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.ap); + const PG = b.axis("PG", cfg.pg); + const A = b.axis("A", cfg.ap * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, cfg.ap * cfg.pg).muli(op.fetch(active, .{})), .{}); + const entries = try selectMetadata(b, t.table, own, S); + const pages = try validIndices(b, entries, cfg.pages); + const firsts = try b.arange(cfg.ap * cfg.pg, .i32, &.{AP}, .{ .step = cfg.pg }); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{AP}, .addi), .{}); + op = b.tensorOperation(.{}); + const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); + op = b.tensorOperation(.{}); + const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = &.{ T, AP, PG } }), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + b.ret(&.{ read, bounds, mask, keyed }); + } + }.run, +}); + +const Decode = tcl.Kernel(DecodeCfg, .{ + .name = "paged_decode_attention", + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { + b.fuse(); + configure(b, .{ .tokens = cfg.t, .sequences = cfg.t, .b = cfg.t, .i = 1, .a = cfg.a, .chunk = cfg.a, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, .pages = cfg.pages, .pg = cfg.pg, .scale = cfg.scale, .decode = true }); + const T = b.axis("T", cfg.t); + const A = b.axis("A", cfg.a); + const AP = b.axis("AP", @divExact(cfg.a, cfg.pg)); + const Hkv = b.axis("Hkv", cfg.hkv); + const G = b.axis("G", cfg.g); + const D = b.axis("D", cfg.d); + const P = b.axis("P", cfg.pages); + const PG = b.axis("PG", cfg.pg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ T, Hkv, G, D } }, + .pages = .{ .dtype = .i32, .axes = &.{ T, AP } }, + .bounds = .{ .dtype = .i32, .axes = &.{T} }, + .mask = .{ .dtype = .i32, .axes = &.{ T, A } }, + .keyed = .{ .dtype = .f32, .axes = &.{T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, + }); + const sink = if (cfg.use_sink) t.sink else null; + const groups = @divExact(cfg.t, cfg.group); + if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); + const NG = b.axis("NG", groups); + const TG = b.axis("TG", @divExact(cfg.t, groups)); + var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); + { + const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); + const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); + const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); + const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); + const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); + const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); + } + b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); + } + }.run, +}); + +const unmasked_group_tokens = 4; + +/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. +fn decodeGroups(tokens: i64, keys: i64) i64 { + var groups: i64 = 1; + while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { + groups += 1; + while (@rem(tokens, groups) != 0) groups += 1; + } + return groups; +} + +/// `keyed` scales the softmax: zero for tokens without keys. +fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const AP = pages.axes[1]; + const P = k_cache.axes[0]; + const PG = k_cache.axes[1]; + const keys: []const tcl.Axis = &.{ AP, PG }; + const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); + var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { + k = try maskCacheValues(b, k, key_mask); + v = try maskCacheValues(b, v, key_mask); + } + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const p = if (sink) |logits| blk: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); + op = b.tensorOperation(.{}); + const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); + op = b.tensorOperation(.{}); + break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + } else blk: { + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); +} + +const head_keys = 4096; +const tail_chunk_keys = 8192; + +const HybridCfg = struct { + t: i64, + s: i64, + /// Head pages, tail chunks and pages per tail chunk. + hp: i64, + nc: i64, + cp: i64, + pg: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + scale: f32, + is_causal: bool, +}; + +fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { + const pg = k_cache.dim(.k_chunk); + const t = q.dim(.b); + const s = parameters.seq_lens.dim(.b); + const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; + const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; + const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; + const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); + const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), + .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), + .hbounds = zml.Shape.init(.{ .t = t }, .i32), + .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), + .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), + .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), + .limits = zml.Shape.init(.{ .t = t }, .i32), + .keyed = zml.Shape.init(.{ .t = t }, .f32), + }, .{ .cfg = cfg }); + return HybridDecode.call(.{ + .q = q.convert(.bf16), + .hpages = meta.hpages, + .hmask = meta.hmask, + .hbounds = meta.hbounds, + .tpages = meta.tpages, + .tmask = meta.tmask, + .tbounds = meta.tbounds, + .limits = meta.limits, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the +/// longest token; `limits` is the visible key count. +const HybridMeta = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); + const HP = b.axis("HP", cfg.hp); + const RP = b.axis("RP", cfg.nc * cfg.cp); + const NC = b.axis("NC", cfg.nc); + const CP = b.axis("CP", cfg.cp); + const HA = b.axis("HA", cfg.hp * cfg.pg); + const CK = b.axis("CK", cfg.cp * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); + + // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read + // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. + // Remove once it is exact on RNGD. + const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + op = b.tensorOperation(.{}); + const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); + // Trips never exceed nc: `limit` is at most the table's keys. + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); + { + const reads, const masks, const counts, const seen = loop.carried; + const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); + const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); + try loop.yield(.{ + try b.indexWrite(reads, loop.iv, chunk.pages), + try b.indexWrite(masks, loop.iv, chunk.mask), + try b.indexWrite(counts, loop.iv, chunk.bounds), + any, + }); + } + const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); + b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + } + }.run, +}); + +/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. +fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { + const T = own.axes[0]; + const S = own.axes[1]; + const PG = b.axis("PG", cfg.pg); + const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); + var op = b.tensorOperation(.{}); + const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); + op = b.tensorOperation(.{}); + const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + const to: []const tcl.Axis = &.{ T, cols, PG }; + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); + return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; +} + +const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; + +const HybridDecode = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_decode_attention", + .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + b.fuse(); + const a = hybridAxes(b, cfg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, + .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, + .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, + .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, + .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, + .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, + .limits = .{ .dtype = .i32, .axes = &.{a.T} }, + .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + }); + const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); + b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); + } + }.run, +}); + +const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; + +fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { + return .{ + .T = b.axis("T", cfg.t), + .HP = b.axis("HP", cfg.hp), + .HA = b.axis("HA", cfg.hp * cfg.pg), + .NC = b.axis("NC", cfg.nc), + .CP = b.axis("CP", cfg.cp), + .CK = b.axis("CK", cfg.cp * cfg.pg), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; +} + +fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { + const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); + { + const max, const sum, const out = loop.carried; + const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); + const bounds = try b.indexRead(tbounds, loop.iv); + const pages = try b.indexRead(tpages, loop.iv); + const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); + try loop.yield(.{ merged.max, merged.sum, merged.out }); + } + return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; +} + +fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { + const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); + const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const keys: []const tcl.Axis = &.{ a.HP, a.PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); + const alpha = try expDifference(b, tail.max, max); + op = b.tensorOperation(.{}); + const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); + op = b.tensorOperation(.{}); + const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); +} + +fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const keys = mask.axes[1..]; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); + const exps = try maskedExp(b, masked, max, mask); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); + const alpha = try expDifference(b, x.max, max); + const beta = try expDifference(b, y.max, max); + op = b.tensorOperation(.{}); + const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); + op = b.tensorOperation(.{}); + const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +const Cfg = struct { + tokens: i64, + sequences: i64, + /// Query blocks and query rows per block. + b: i64, + i: i64, + /// Keys per block (padded to whole chunks) and keys per chunk. + a: i64, + chunk: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + decode: bool, + use_sink: bool = false, + is_causal: bool = true, + window: i32 = -1, +}; + +/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. +fn configure(b: *tcl.Builder, cfg: Cfg) void { + const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; + if (cfg.decode) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "Gather", + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, + .dma_preference = 1.2, + }); +} + +const Axes = struct { + T: tcl.Axis, + S: tcl.Axis, + O: tcl.Axis, + AP: tcl.Axis, + B: tcl.Axis, + I: tcl.Axis, + Hkv: tcl.Axis, + G: tcl.Axis, + D: tcl.Axis, + P: tcl.Axis, + PG: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.tokens), + .S = b.axis("S", cfg.sequences), + .O = b.axis("O", cfg.sequences + 1), + .AP = b.axis("AP", @divExact(cfg.a, cfg.pg)), + .B = b.axis("B", cfg.b), + .I = b.axis("I", cfg.i), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; + } +}; + +const NativeInputs = struct { + q: builder.Tensor, + pages: builder.Tensor, + seq_lens: builder.Tensor, + query_offsets: builder.Tensor, + k_cache: builder.Tensor, + v_cache: builder.Tensor, + sink: builder.Tensor, +}; + +fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .pages = .{ .dtype = .i32, .axes = &.{ a.S, a.AP } }, + .seq_lens = .{ .dtype = .i32, .axes = &.{a.S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{a.O} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .sink = .{ .dtype = .f32, .axes = &.{ a.Hkv, a.G } }, + }); + return .{ .q = t.q, .pages = t.pages, .seq_lens = t.seq_lens, .query_offsets = t.query_offsets, .k_cache = t.k_cache, .v_cache = t.v_cache, .sink = t.sink }; +} + +const Prepared = struct { + q: builder.Tensor, + /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). + pages: builder.Tensor, + pick: ?builder.Tensor, + qpos: builder.Tensor, + kvlen: builder.Tensor, + row: builder.Tensor, + active: builder.Tensor, + /// Per page row: the first query position and the keys. + first: builder.Tensor, + keys: builder.Tensor, + + /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash + /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such + /// products compile. + fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { + const pick = self.pick orelse return pages; + return selectMetadata(b, pages, pick, a.S); + } +}; + +fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(ends, .{}).subi(op.fetch(positions, .{})), .{}); + op = b.tensorOperation(.{}); + const passed = try op.commit(op.where(distance, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(passed, .{}).reduce(&.{axis}, .addi), .{}); +} + +fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64) tcl.FinishError!builder.Tensor { + const source = try b.arange(count, .i32, &.{axis}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(indices, .{}).subi(op.fetch(source, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); +} + +// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for +// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, +// repro_i32_slice_odd_offset.py. Remove once both compile. +fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); + return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); +} + +/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would +/// need alignment staging at every invocation. +fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { + const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); + const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); + const starts = try selectMetadata(b, t.query_offsets, try selector(b, start_indices, a.O, cfg.sequences + 1), a.O); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, end_indices, a.O, cfg.sequences + 1), a.O); + var op = b.tensorOperation(.{}); + const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); + const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); + const seq_of_token = try countBefore(b, ends, tokens, a.S); + // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. + op = b.tensorOperation(.{}); + const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const active = try op.commit(op.where(token_tail, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const token_seq = try op.commit(op.fetch(seq_of_token, .{}).binary(.mini, cfg.sequences - 1), .{}); + if (cfg.decode) { + op = b.tensorOperation(.{}); + const live = try op.commit(op.where(qlen, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const lengths = try op.commit(op.fetch(t.seq_lens, .{}).muli(op.fetch(live, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.fetch(lengths, .{}).subi(1), .{}); + op = b.tensorOperation(.{}); + const safe_start = try op.commit(op.fetch(starts, .{}).binary(.mini, cfg.tokens - 1), .{}); + const q = if (cfg.tokens == 1) t.q else try b.gather(t.q, safe_start, a.T, &.{ a.S, a.Hkv, a.G, a.D }, .{}); + return .{ + .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), + .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .pick = null, + .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), + .kvlen = try b.reshape(lengths, &.{a.B}, .{}), + .row = token_seq, + .active = active, + .first = try b.reshape(qpos, &.{a.B}, .{}), + .keys = try b.reshape(lengths, &.{a.B}, .{}), + }; + } + + const shift = std.math.log2_int(u64, @intCast(cfg.i)); + op = b.tensorOperation(.{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); + op = b.tensorOperation(.{}); + const block_start = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})), .{}); + const block_ids = try b.arange(cfg.b, .i32, &.{a.B}, .{}); + const seq_of_block = try countBefore(b, block_end, block_ids, a.S); + op = b.tensorOperation(.{}); + const beyond = try op.commit(op.fetch(seq_of_block, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const live_block = try op.commit(op.where(beyond, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const seq = try op.commit(op.fetch(seq_of_block, .{}).binary(.mini, cfg.sequences - 1), .{}); + const pick = try selector(b, seq, a.S, cfg.sequences); + const block_begin = try selectMetadata(b, block_start, pick, a.S); + const begin = try selectMetadata(b, starts, pick, a.S); + const query_length = try selectMetadata(b, qlen, pick, a.S); + const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); + const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); + op = b.tensorOperation(.{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); + op = b.tensorOperation(.{}); + const remaining = try op.commit(op.fetch(qi, .{}).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const inside = try op.commit(op.where(remaining, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const live = try op.commit(op.fetch(inside, .{}).muli(op.fetch(live_block, .{})), .{}); + op = b.tensorOperation(.{}); + const qidx = try op.commit(op.fetch(qi, .{}).addi(op.fetch(begin, .{})).muli(op.fetch(live, .{})), .{}); + const q = try b.gather(t.q, qidx, a.T, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}); + op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(qi, .{}).addi(op.fetch(key_length, .{})).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); + op = b.tensorOperation(.{}); + const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); + + const token_pick = try selector(b, token_seq, a.S, cfg.sequences); + const token_begin = try selectMetadata(b, starts, token_pick, a.S); + const token_block = try selectMetadata(b, block_start, token_pick, a.S); + op = b.tensorOperation(.{}); + const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + op = b.tensorOperation(.{}); + const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); + return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; +} + +fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { + if (cfg.decode and cfg.tokens == 1) return b.reshape(output, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const R = b.axis("R", cfg.b * cfg.i); + const rows = try b.reshape(output, &.{ R, a.Hkv, a.G, a.D }, .{}); + const ordered = try b.gather(rows, prepared.row, R, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const op = b.tensorOperation(.{ .tactic = .elementwise }); + // Establish the full data layout before broadcasting the I32 row mask. + const values = op.fetch(ordered, .{}); + return op.commit(op.where(op.fetch(prepared.active, .{ .broadcast_to = ordered.axes }), .ne, 0, values, 0.0), .{ .dtype = .bf16 }); +} + +const score_min = -std.math.floatMax(f32); + +fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.FinishError!struct { safe: builder.Tensor, valid: builder.Tensor } { + var op = b.tensorOperation(.{}); + const safe = try op.commit(op.fetch(indices, .{}).binary(.maxi, 0).binary(.mini, count - 1), .{}); + op = b.tensorOperation(.{}); + const nonnegative = try op.commit(op.fetch(indices, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const valid = try op.commit(op.fetch(indices, .{}).muli(-1).addi(count).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(nonnegative, .{})), .{}); + return .{ .safe = safe, .valid = valid }; +} + +/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. +fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); +} + +/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. +fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const data = op.fetch(values, .{}); + return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); +} + +fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; + const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; + // Elementwise operations require their first read to cover the output + // axes, and each VE instruction must consume the current accumulator. + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, read), .{}); + op = b.tensorOperation(.{}); + const query = try op.commit(op.fetch(qpos, read), .{}); + op = b.tensorOperation(.{}); + var mask = try op.commit(op.fetch(valid_keys, read), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + if (cfg.is_causal) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).subi(op.fetch(pos, .{})).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).subi(op.fetch(query, .{})).addi(cfg.window).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + return mask; +} + +fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { + const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, scores_axes).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); +} + +fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); + // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); +} + +fn expDifference(b: *tcl.Builder, x: builder.Tensor, maximum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const m = op.fetch(maximum, .{}); + return op.commit(op.fetch(x, .{}).subf(m).exp(), .{}); +} + +fn safeDenominator(b: *tcl.Builder, sum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const s = op.fetch(sum, .{}); + return op.commit(op.where(s, .gt, 0, s, 1.0), .{}); +} + +const OneShot = tcl.Kernel(Cfg, .{ + .name = "attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const A = b.axis("A", cfg.a); + const Nb = b.axis("Nb", cfg.pages * cfg.pg); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); + const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); + var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); + const expanded_slots = try b.reshape(flat_slots, &.{ a.B, A }, .{}); + slot_op = b.tensorOperation(.{}); + const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); + const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); + slot_op = b.tensorOperation(.{}); + const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); + const valid_length = if (cfg.decode) prepared.kvlen else null; + const position = try b.arange(cfg.a, .i32, &.{A}, .{}); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.gather(kc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const v = try maskCacheValues(b, try b.gather(vc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); + var op = b.tensorOperation(.{}); + const p = if (cfg.decode) compact: { + const exp_sink = if (cfg.use_sink) sink: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); + break :sink try expDifference(b, t.sink, maximum); + } else null; + op = b.tensorOperation(.{}); + const scores = op.fetch(masked, .{}); + var maximum = scores.reduce(&.{A}, .maxf); + if (cfg.use_sink) maximum = maximum.maxf(op.fetch(t.sink, .{})); + const e = scores.subf(maximum).exp(); + var sum = e.reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); + // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. + op = b.tensorOperation(.{}); + break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); + } else tiled: { + // Separate reductions schedule better for multi-query blocks with the current compiler. + var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); + if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); + const maximum = try op.commit(m, .{}); + const e = try maskedExp(b, masked, maximum, mask); + const exp_sink = if (cfg.use_sink) try expDifference(b, t.sink, maximum) else null; + op = b.tensorOperation(.{}); + var sum = op.fetch(e, .{}).reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const denominator = try safeDenominator(b, try op.commit(sum, .{})); + op = b.tensorOperation(.{}); + break :tiled try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + const output = try op.commit(op.contract(v, p, &.{ a.B, a.I, a.Hkv, a.G, a.D }), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +const Flash = tcl.Kernel(Cfg, .{ + .name = "flash_attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const C = b.axis("C", @divExact(cfg.a, cfg.chunk)); + const Ck = b.axis("Ck", cfg.chunk); + const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const rows = prepared.pages.axes[0]; + const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); + const page_op = b.tensorOperation(.{}); + const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); + const stats = &.{ a.Hkv, a.G, a.B, a.I }; + const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; + var op = b.tensorOperation(.{}); + const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); + // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides + // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. + const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { + op = b.tensorOperation(.{}); + const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); + // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per + // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index + // compiles as data. + const trip = try b.arange(C.size, .i32, &.{C}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); + break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; + } else null; + const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); + { + const sum_acc, const qkv_acc, const max_acc = loop.carried; + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + var row_pages: builder.Tensor = undefined; + var position: builder.Tensor = undefined; + var first_keys: builder.Tensor = undefined; + if (window) |w| { + const chunk_index = try b.indexRead(w.chunks, loop.iv); + op = b.tensorOperation(.{}); + const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); + // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower + // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove + // once batched gathers of several rows compile. + row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); + op = b.tensorOperation(.{}); + const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); + const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } else { + row_pages = try b.indexRead(pages, loop.iv); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } + const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); + const idx = indices.safe; + op = b.tensorOperation(.{}); + const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); + const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); + op = b.tensorOperation(.{}); + const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); + op = b.tensorOperation(.{}); + const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); + const rescale = try expDifference(b, max_acc, max_new); + const e = try maskedExp(b, masked, max_new, mask); + op = b.tensorOperation(.{}); + const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); + op = b.tensorOperation(.{}); + const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); + op = b.tensorOperation(.{}); + // Materialize the narrowing conversion through the vector engine. + const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); + op = b.tensorOperation(.{}); + const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); + try loop.yield(.{ sum_new, qkv_new, max_new }); + } + + var sum = loop.results[0]; + var numerator = loop.results[1]; + if (cfg.use_sink) { + op = b.tensorOperation(.{}); + const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); + const alpha = try expDifference(b, loop.results[2], maximum); + const exp_sink = try expDifference(b, t.sink, maximum); + op = b.tensorOperation(.{}); + sum = try op.commit(op.fetch(sum, .{}).mulf(op.fetch(alpha, .{})).addf(op.fetch(exp_sink, .{})), .{}); + op = b.tensorOperation(.{}); + numerator = try op.commit(op.fetch(numerator, .{}).mulf(op.fetch(alpha, .{})), .{}); + } + const denominator = try safeDenominator(b, sum); + op = b.tensorOperation(.{}); + const output = try op.commit(op.fetch(numerator, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +test { + std.testing.refAllDecls(attention_test); +} + +const attention_test = struct { + pub const Options = struct { + is_causal: bool = true, + sliding_window: i32 = -1, + scale: ?f32 = null, + sink: ?[]const f32 = null, + }; + + pub fn reference(allocator: std.mem.Allocator, q: zml.Slice, k: zml.Slice, v: zml.Slice, table: []const i32, lengths: []const i32, starts: []const i32, opts: Options) !zml.Slice { + const bf16 = zml.floats.BFloat16; + const hkv: usize = @intCast(q.shape.dim(.hkv)); + const groups: usize = @intCast(q.shape.dim(.hg)); + const d: usize = @intCast(q.shape.dim(.hd)); + const page: usize = @intCast(k.shape.dim(.k_chunk)); + const pages: usize = @intCast(k.shape.dim(.page)); + const max_pages = table.len / lengths.len; + const scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(d))); + const queries = q.constItems(bf16); + const keys = k.constItems(bf16); + const values = v.constItems(bf16); + const result = try zml.Slice.alloc(allocator, q.shape.withDtype(.f32)); + errdefer result.free(allocator); + const out = result.items(f32); + @memset(out, 0); + const scores = try allocator.alloc(f32, max_pages * page); + defer allocator.free(scores); + + for (lengths, 0..) |length, seq| { + const qlen = starts[seq + 1] - starts[seq]; + for (@intCast(starts[seq])..@intCast(starts[seq + 1])) |token| { + const qpos = length - qlen + @as(i32, @intCast(token)) - starts[seq]; + if (qpos < 0) continue; + for (0..hkv) |h| for (0..groups) |g| { + const offset = ((token * hkv + h) * groups + g) * d; + const sink = if (opts.sink) |s| s[h * groups + g] else -std.math.inf(f32); + var maximum = sink; + @memset(scores, -std.math.inf(f32)); + for (0..@intCast(length)) |key| { + const pos: i32 = @intCast(key); + if (opts.is_causal and pos > qpos) continue; + if (opts.sliding_window > 0 and qpos - pos >= opts.sliding_window) continue; + const physical = table[seq * max_pages + key / page]; + if (physical < 0 or physical >= pages) continue; + const kv_offset = ((@as(usize, @intCast(physical)) * page + key % page) * hkv + h) * d; + var dot: f32 = 0; + for (0..d) |i| dot += queries[offset + i].toF32() * keys[kv_offset + i].toF32(); + scores[key] = dot * scale; + maximum = @max(maximum, scores[key]); + } + if (maximum == -std.math.inf(f32)) continue; + var denominator = @exp(sink - maximum); + for (scores[0..@intCast(length)]) |score| denominator += @exp(score - maximum); + for (0..@intCast(length)) |key| { + if (scores[key] == -std.math.inf(f32)) continue; + const physical: usize = @intCast(table[seq * max_pages + key / page]); + const kv_offset = ((physical * page + key % page) * hkv + h) * d; + const probability = @exp(scores[key] - maximum) / denominator; + for (0..d) |i| out[offset + i] += probability * values[kv_offset + i].toF32(); + } + }; + } + } + return result; + } + + test "paged attention host reference scale and sink normalization" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(1)); + const keys = [_]bf16{ .fromF32(0), .fromF32(1) }; + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: f32 }{ + .{ .opts = .{ .is_causal = false, .scale = 0 }, .expected = 3 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)) }, .expected = 3.5 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)), .sink = &.{0} }, .expected = 2.8 }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + for (result.constItems(f32)[0..2]) |value| try std.testing.expectApproxEqAbs(case.expected, value, 1e-6); + try std.testing.expectEqual(@as(f32, 0), result.constItems(f32)[2]); + } + } + + pub fn expectClose(expected: zml.Slice, actual: zml.Slice) !void { + try std.testing.expectEqualSlices(i64, expected.shape.dims(), actual.shape.dims()); + for (expected.constItems(f32), 0..) |want, i| { + const got = floatAt(actual, i); + try std.testing.expect(std.math.isFinite(got)); + std.testing.expectApproxEqAbs(want, got, 3e-2 + 3e-2 * @abs(want)) catch |err| { + std.log.err("paged attention mismatch at element {d}: expected {d}, got {d}", .{ i, want, got }); + return err; + }; + } + } + + fn floatAt(slice: zml.Slice, index: usize) f32 { + return switch (slice.shape.dtype()) { + .bf16 => slice.constItems(zml.floats.BFloat16)[index].toF32(), + .f16 => @floatCast(slice.constItems(f16)[index]), + .f32 => slice.constItems(f32)[index], + else => unreachable, + }; + } + + test "paged attention host reference windows sinks and padding" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(0)); + const keys: [2]bf16 = @splat(.fromF32(0)); + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: [3]f32 }{ + .{ .opts = .{}, .expected = .{ 2, 3, 0 } }, + .{ .opts = .{ .sliding_window = 1 }, .expected = .{ 2, 4, 0 } }, + .{ .opts = .{ .is_causal = false, .sliding_window = 1 }, .expected = .{ 3, 4, 0 } }, + .{ .opts = .{ .sink = &.{0} }, .expected = .{ 1, 2, 0 } }, + .{ .opts = .{ .sink = &.{-std.math.inf(f32)} }, .expected = .{ 2, 3, 0 } }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + try std.testing.expectEqualSlices(f32, &case.expected, result.constItems(f32)); + } + } +}; + +const TestCase = struct { + prefill: bool, + seqs: []const [2]i32, + padding: i64 = 0, + opts: attention_test.Options = .{}, + mask_all: bool = false, + padding_page: i32 = -1, + poison: enum { none, k, v, both } = .none, + poison_value: f32 = std.math.nan(f32), +}; + +const TestDims = struct { + batch: usize = 4, + hkv: usize = 2, + g: usize = 4, + d: usize = 64, + page: usize = 16, + uniform: bool = false, + uniform_scores: bool = false, + spikes: []const usize = &.{}, + dtype: zml.DataType = .bf16, +}; + +test "tcl paged attention unit scale and dtype boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{.{ .prefill = true, .seqs = &@as([4][2]i32, @splat(.{ 32, 256 })), .opts = .{ .scale = 1.0 } }}, .{}); + inline for (.{ .f16, .f32 }) |dtype| { + try runCases(2, 8, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 17 }, .{ 1, 1 }, .{ 1, 3 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sink = &.{ 0, -std.math.inf(f32) } } }}, .{ .hkv = 1, .g = 2, .d = 32, .dtype = dtype }); + } +} + +test "tcl paged attention single decode row needs no rearrangement" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 1 }, + }, .{ .batch = 1 }); + inline for (.{ 257, 320 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 16 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 8, .g = 4, .d = 128 }); + } + inline for (.{ 5, 1025 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 48 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 1, .g = 1, .d = 32, .page = 48 }); + } +} + +test "tcl paged attention packs inactive leading sequences and odd batches" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 17 }, .{ 1, 63 }, .{ 0, 23 }, .{ 1, 65 } }, .padding = 2 }, + .{ .prefill = true, .seqs = &.{ .{ 0, 17 }, .{ 33, 64 }, .{ 0, 23 }, .{ 2, 3 } }, .padding = 3 }, + }, .{}); + try runCases(2, 8, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 9 }, .{ 1, 17 }, .{ 1, 1 }, .{ 0, 0 } }, .padding = 1 }, + }, .{ .batch = 3, .hkv = 1, .g = 2, .d = 32 }); +} + +test "tcl paged attention masks inactive decode rows across all heads" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(32, 64, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, 511 }, .{ 1, 504 }, .{ 1, 497 }, .{ 1, 490 }, .{ 0, 483 }, .{ 1, 476 }, .{ 1, 469 }, .{ 1, 462 } }, + .padding = 1, + }}, .{ .batch = 8 }); +} + +test "tcl paged attention invalid pages cannot wrap into slot zero" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = 1 << 28 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = std.math.maxInt(i32) }, + }; + try runCases(2, 8, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention emits decode and prefill kernels with and without sinks" { + inline for (.{ false, true }) |decode| { + inline for (.{ false, true }) |use_sink| { + const cfg: Cfg = .{ .tokens = if (decode) 2 else 8, .sequences = 2, .b = if (decode) 2 else 3, .i = if (decode) 1 else prefill_block, .a = 32, .chunk = 16, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .decode = decode, .use_sink = use_sink }; + const one_shot = try OneShot.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(one_shot); + const flash = try Flash.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(flash); + for ([_][]const u8{ one_shot, flash }) |ir| { + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, ir, "!tcl.logical") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); + + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); + defer std.testing.allocator.free(decode); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); + try std.testing.expect(std.mem.indexOf(u8, decode, "propagate_sparse_axis_from_op = \"Gather\"") != null); +} + +test "tcl paged attention excludes poisoned unused cache entries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 17 }, .{ 1, 31 }, .{ 0, 0 }, .{ 0, 23 } }; + try runCases(8, 32, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .k }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .v }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = -std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = 1e30 }, + // A sink selects OneShot's compact decode softmax instead of Decode. + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .opts = .{ .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .mask_all = true, .padding_page = std.math.maxInt(i32) }, + .{ .prefill = false, .seqs = &.{ .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 4, .poison = .both }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const mixed = [_][2]i32{ .{ 37, 65 }, .{ 0, 19 }, .{ 1, 17 }, .{ 4, 2 } }; + try runCases(8, 32, &.{ + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .k }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .v }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .both, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const boundary = [_][2]i32{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 17 } }; + try runCases(257, 1028, &.{ + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both }, + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both, .poison_value = std.math.inf(f32), .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 17, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(86, 344, &.{.{ + .prefill = true, + .seqs = &.{ .{ 3, 4033 }, .{ 1, 4032 }, .{ 0, 0 }, .{ 1, 4031 } }, + .padding = 1, + .poison = .both, + .poison_value = -std.math.inf(f32), + .opts = .{ .sliding_window = 49 }, + }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention runs on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + inline for (.{ 16, 320 }) |max_pages| { + const cases = if (max_pages == 16) [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 200 }, .{ 1, 77 }, .{ 20, 20 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 250 }, .{ 1, 3 }, .{ 1, 64 }, .{ 1, 129 } } }, + } else [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4500 }, .{ 1, 77 }, .{ 70, 70 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 5000 }, .{ 1, 3 }, .{ 1, 4097 }, .{ 1, 129 } } }, + }; + try runCases(max_pages, 2 * max_pages, &cases, .{}); + } +} + +test "tcl paged attention options and inactive queries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const sinks = [_]f32{ -std.math.inf(f32), -5, 0, 2, 8, -1, 1, 20 }; + const disabled: [8]f32 = @splat(-std.math.inf(f32)); + const mixed = [4][2]i32{ .{ 37, 200 }, .{ 0, 0 }, .{ 1, 77 }, .{ 20, 20 } }; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 0 } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 1 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + }; + try runCases(16, 32, &cases, .{}); +} + +test "tcl paged attention chunk boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 4095 } }, .padding = 7, .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, 10 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 17, .sink = &.{ -std.math.inf(f32), 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 4097 })), .mask_all = true, .opts = .{ .sink = &.{ -std.math.inf(f32), 0 } } }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(256, 512, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 2 }}, .{ .hkv = 1, .g = 1, .d = 32 }); + try runCases(86, 172, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4079 }, .{ 1, 4080 }, .{ 1, 4081 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 49 } }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention preserves constant values across chunks" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 1, .sink = &.{0} } }, + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 77 }, .{ 2, 2 } }, .padding = 3 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32, .uniform = true }); +} + +test "tcl paged attention chunked decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention hybrid decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(2049, 2101, &.{ + .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, + }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); +} + +test "tcl paged attention flash decode of many tokens on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); + try runCases(2049, 2600, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2700, &.{ + .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, + }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); +} + +test "tcl paged attention uniform probabilities on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); +} + +fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const TestCase, comptime dims: TestDims) !void { + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + const hkv = dims.hkv; + const g = dims.g; + const d = dims.d; + const page = dims.page; + const batch = dims.batch; + var prng: std.Random.DefaultPrng = .init(7); + const r = prng.random(); + const kc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(kc); + const vc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(vc); + const k_t: Tensor = .init(.{ .page = num_pages, .k_chunk = page, .hkv = hkv, .hd = d }, dims.dtype); + + for (cases) |case| { + var tokens: i64 = case.padding; + for (case.seqs[0..batch]) |s| tokens += s[0]; + const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); + const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); + const Mod = struct { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { + return paged.pagedAttention(p, q, k, v, opts); + } + }; + const opts: AttentionOptions = .{ + .mask = .{ .standard = .{ + .causal = case.opts.is_causal, + .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, + } }, + .scale = case.opts.scale, + .sink = if (case.opts.sink != null) Tensor.init(.{ .h = hkv * g }, .f32) else null, + }; + const v_t: Tensor = .init(k_t.shape(), dims.dtype); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ params, q_t, k_t, v_t, opts }, platform, .{}); + defer exe.deinit(); + + const q = try allocator.alloc(bf16, @intCast(tokens * hkv * g * d)); + defer allocator.free(q); + for (q) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + var table: [batch][max_pages]i32 = @splat(@splat(case.padding_page)); + var seq_lens: [batch]i32 = undefined; + var starts: [batch + 1]i32 = undefined; + starts[0] = 0; + var next_page: i32 = 0; + for (case.seqs[0..batch], 0..) |s, seq| { + seq_lens[seq] = s[1]; + starts[seq + 1] = starts[seq] + s[0]; + if (!case.mask_all) { + for (0..@intCast(std.math.divCeil(i32, s[1], page) catch unreachable)) |p| { + table[seq][p] = @mod(next_page * 7, @as(i32, num_pages)); + next_page += 1; + } + } + } + + for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (dims.spikes.len > 0) { + for (case.seqs[0..batch], 0..) |seq, row| { + for (0..@intCast(seq[1])) |key| { + const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); + const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; + @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); + } + } + } + if (case.poison != .none) { + const live = try allocator.alloc(bool, num_pages * page); + defer allocator.free(live); + @memset(live, false); + for (case.seqs[0..batch], 0..) |s, seq| { + if (s[0] == 0) continue; + for (0..@intCast(s[1])) |key| { + const physical = table[seq][key / page]; + if (physical < 0 or physical >= num_pages) continue; + live[@as(usize, @intCast(physical)) * page + key % page] = true; + } + } + const poison = bf16.fromF32(case.poison_value); + for (live, 0..) |valid, slot| { + if (valid) continue; + const start = slot * hkv * d; + if (case.poison == .k or case.poison == .both) @memset(kc[start..][0 .. hkv * d], poison); + if (case.poison == .v or case.poison == .both) @memset(vc[start..][0 .. hkv * d], poison); + } + } + var bk = try testBuffer(platform, k_t.shape(), kc); + defer bk.deinit(); + var bv = try testBuffer(platform, k_t.shape(), vc); + defer bv.deinit(); + var bq = try testBuffer(platform, q_t.shape(), q); + defer bq.deinit(); + var bp: zml.Bufferized(paged.Parameters) = .{ + .block_table = try .fromBytes(io, platform, params.block_table.shape(), std.mem.sliceAsBytes(&table)), + .seq_lens = try .fromBytes(io, platform, params.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), + }; + defer zml.Buffer.deinitAll(paged.Parameters, &bp); + var opts_d: zml.Bufferized(AttentionOptions) = .{ + .mask = .{ .standard = {} }, + .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, + }; + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); + var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); + defer result.deinit(); + const host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const expected = try attention_test.reference( + allocator, + .initConst(q_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(q)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(kc)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(vc)), + @as([*]const i32, @ptrCast(&table))[0 .. batch * max_pages], + &seq_lens, + &starts, + case.opts, + ); + defer expected.free(allocator); + try std.testing.expectEqual(dims.dtype, host.shape.dtype()); + attention_test.expectClose(expected, host) catch |err| { + std.log.err("TCL attention fixture: {any}", .{case}); + return err; + }; + const active_elements = if (case.mask_all) 0 else @as(usize, @intCast(starts[batch])) * hkv * g * d; + for (active_elements..@intCast(tokens * hkv * g * d)) |index| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, index)); + for (case.seqs[0..batch], 0..) |seq, index| { + const empty_queries: usize = @intCast(@max(0, seq[0] - seq[1])); + const begin: usize = @as(usize, @intCast(starts[index])) * hkv * g * d; + for (begin..begin + empty_queries * hkv * g * d) |element| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, element)); + } + } +} + +fn testBuffer(platform: *const zml.Platform, shape: zml.Shape, values: []const zml.floats.BFloat16) !zml.Buffer { + const slice = try zml.Slice.alloc(std.testing.allocator, shape); + defer slice.free(std.testing.allocator); + switch (shape.dtype()) { + .bf16 => @memcpy(slice.items(zml.floats.BFloat16), values), + .f16 => for (slice.items(f16), values) |*out, value| { + out.* = @floatCast(value.toF32()); + }, + .f32 => for (slice.items(f32), values) |*out, value| { + out.* = value.toF32(); + }, + else => unreachable, + } + return .fromBytes(std.testing.io, platform, shape, slice.bytes); +} diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel new file mode 100644 index 00000000..eff7d096 --- /dev/null +++ b/zml/kernels/paged_attention/triton/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "sparse_unified_attention.zig", + "triton.zig", + "unified_attention.zig", + "unified_attention_oneapi.zig", + ], +) diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md new file mode 100644 index 00000000..37aba88b --- /dev/null +++ b/zml/kernels/paged_attention/triton/README.md @@ -0,0 +1,57 @@ +# Triton paged attention + +This backend is automatically selected for CUDA, ROCm, and oneAPI. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[Triton's wrapper](triton.zig) uses split KV caches and supports causal and +non-causal attention, a left sliding window, explicit key indices, and softmax +sinks. Query heads are grouped by KV head. Keep complete GQA groups together +when sharding `.hkv`, with matching query and cache ownership and complete +`.hg` and `.hd` axes. Each local computation needs its full key range and the +matching sequence metadata. + +For standard masks, the wrapper chooses between a 2D kernel and a 3D split-key +kernel followed by segment reduction. Prefill uses the 2D path; all-decode +batches select a path based on batch size and KV head count. Explicit indices +use the dedicated sparse path described below. oneAPI forces split-key execution for +all-decode batches and has a specialized path for standard masks. Segment +reduction combines softmax maxima, exponential sums, and weighted values to +preserve one normalization over the key range. + +Set `is_prefill = true` for mixed prefill/decode batches. `max_seqlen_q` guides +kernel configuration. These options do not change logical query positions. + +The shared tests in [paged_attention.zig](../paged_attention.zig) exercise +`bf16`, mixed prefill/decode, shared prefix pages, partial final KV pages, +trailing inactive sequences, and option-specific comparisons. Kernel selection +and numerical behavior depend on the target; these cases do not establish +support for every shape or dtype. + +## Sparse attention + +`.indices` mask dispatches to a dedicated sparse kernel in +`sparse_unified_attention.zig`. +The `i32` tensor has shape `{ .b = T, .topk = K }`, with one list per packed +query shared across heads. Each entry is a logical position within that query's +sequence, including its cached prefix. The kernel translates each selected +position through the existing page table; it does not compute top-k selection. +Negative and out-of-sequence entries are ignored, including holes before valid +tail entries. Duplicate entries count repeatedly. Indices replace the standard +mask, so they do not implicitly enforce causality. Empty selections return zero; +optional attention sinks contribute only to the softmax denominator. + +Prefill processes one query and one KV-head group per program and writes the +normalized result directly. Decode uses 16- or 32-position gather tiles and an +occupancy-based split count bounded by selection length. Single-token decode +addresses sequence slots directly; other query layouts use packed-query lookup. +Inactive sequence slots and unused query capacity are skipped. Only segments +covering selection slots launch. Small empty segments skip Q and KV loading, +and larger segments skip wholly empty tiles. Known strides and scale are +specialized, and sparse gathers use the default cache policy. + +Split-K decode writes FP32 partials and contiguous-head statistics. Its reducer +handles at most 64 features per program, ignores unlaunched and zero-mass +segments, and never reads their unwritten partial outputs. The direct and split +paths use the same online softmax and support non-power-of-two GQA groups and +padded head dimensions. Dense attention retains its existing dispatch and kernels. diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig new file mode 100644 index 00000000..79113f1e --- /dev/null +++ b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig @@ -0,0 +1,326 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +/// Partition selection slots, not logical KV positions. Scratch capacity may be +/// larger than the number of segments that have any selection slots to visit. +pub const SparseSegments = struct { + tiles_per_segment: i64, + span: i64, + launched: i64, + + pub fn init(count: i64, capacity: i64, tile: i64) SparseSegments { + std.debug.assert(count > 0 and capacity > 0 and tile > 0); + const tiles = std.math.divCeil(i64, count, capacity * tile) catch unreachable; + return .{ + .tiles_per_segment = tiles, + .span = tiles * tile, + .launched = std.math.divCeil(i64, count, tiles * tile) catch unreachable, + }; + } +}; + +pub const SparseAttentionConfig = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + num_queries_per_kv: i64, + head_size: i64, + head_size_padded: i64, + block_m: i64, + block_size: i64, + tile_size: i64, + indices_count: i64, + num_segments: i64 = 1, + decode_sequence_grid: bool = false, + use_sinks: bool = false, + scale: f32, + block_table_stride: i64, + query_strides: [3]i64, + key_strides: [4]i64, + value_strides: [4]i64, + indices_strides: [2]i64, +}; + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +/// One program processes all selected keys for one query/KV-head group and write the normalized output. +pub const KernelSparseAttention2dPtr = KernelSparseAttentionPtr(false); + +/// Also known as split-K in the litterature, this 3d variant uses multiple programs process different +/// portions of that same selection list. They write partial results, then a reduction kernels combines +/// them into one normalized output. +/// +/// Mostly useful during decode when there are few queries and a long top-k list. +pub const KernelSparseAttention3dPtr = KernelSparseAttentionPtr(true); + +/// One query and one GQA group per program. The direct specialization writes +/// normalized output; split decode writes FP32 partials for SparseReducePtr. +fn KernelSparseAttentionPtr(comptime split_k: bool) type { + return struct { + pub const Config = SparseAttentionConfig; + pub const Kernel = tri.Kernel(Config, .{ + .name = if (split_k) "kernel_sparse_attention_3d_ptr" else "kernel_sparse_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_ptr", "value_ptr", "indices_ptr", + "block_tables_ptr", "seq_lens_ptr", "query_start_len_ptr", "sink_ptr", + }, + .outputs = if (split_k) &.{ "output", "segm_max", "segm_expsum" } else &.{"output"}, + .run = run, + }); + + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = if (split_k) try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }) else try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + + std.debug.assert(cfg.block_m >= cfg.num_queries_per_kv); + std.debug.assert(split_k or cfg.num_segments == 1); + + const partition = SparseSegments.init(cfg.indices_count, cfg.num_segments, cfg.tile_size); + const program = k.programId(.x); + const num_seqs = k.constant(.i32, cfg.num_seqs); + if (!cfg.decode_sequence_grid) { + const active_queries = k.load(a.query_start_len_ptr.addPtr(num_seqs)); + k.returnIf(program.ge(active_queries)); + } + + const seq = if (cfg.decode_sequence_grid) program else findSeqIdx(k, a.query_start_len_ptr, program, num_seqs, 1, false); + const seq_len = k.load(a.seq_lens_ptr.addPtr(seq)); + const query_start = k.load(a.query_start_len_ptr.addPtr(seq)); + const query_end = k.load(a.query_start_len_ptr.addPtr(seq).addPtr(1)); + if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end)); + const query = if (cfg.decode_sequence_grid) query_start else program; + const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.constant(.i32, 0) else k.programId(.y); + + // Launch the partial tail first, without launching unused capacity. + const slot = if (split_k) k.programId(.z) else k.constant(.i32, 0); + const segment = if (split_k) k.where(slot.eq(0), k.constant(.i32, partition.launched - 1), slot.sub(1)) else slot; + const segment_start = segment.mul(@as(i32, @intCast(partition.span))); + const heads = k.arange(0, cfg.block_m, .i32); + const head = kv_head.mul(@as(i32, @intCast(cfg.num_queries_per_kv))).add(heads); + const head_mask = heads.lt(@as(i32, @intCast(cfg.num_queries_per_kv))); + const dims = k.arange(0, cfg.head_size_padded, .i32); + const dim_mask = dims.lt(@as(i32, @intCast(cfg.head_size))); + const indices_row = a.indices_ptr.addPtr(query.to(.i64).mul(cfg.indices_strides[0])); + + const sink = if (cfg.use_sinks) k.loadOpts(a.sink_ptr.addPtr(head), .{ + .mask = head_mask.bitAnd(segment.eq(0)), + .other = k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32), + }).mul(RCP_LN2) else k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32); + const has_sink = if (cfg.use_sinks) segment.eq(0).bitAnd(head_mask) else k.full(&.{cfg.block_m}, 0, .i1); + const initial_max = k.where(has_sink, sink, k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32)); + const initial_sum = has_sink.to(.f32); + const initial_acc = k.zeros(&.{ cfg.block_m, cfg.head_size_padded }, .f32); + + if (partition.span <= 256) { + const scan_size: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(partition.span))); + const selected = segment_start.add(k.arange(0, scan_size, .i32)); + const scan_mask = selected.lt(cfg.indices_count).bitAnd(selected.lt(segment_start.add(@as(i32, @intCast(partition.span))))); + const positions = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = scan_mask, + .other = k.full(&.{scan_size}, -1, .i32), + }); + const empty = k.sum(scan_mask.bitAnd(positions.ge(0)).bitAnd(positions.lt(seq_len)).to(.i32)).eq(0); + var branch = k.openIfElse(empty, .{}); + { + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); + branch.elseBranch(); + } + { + branch.close(); + } + k.returnIf(empty); + } + + const q_offset = query.to(.i64).mul(cfg.query_strides[0]) + .add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)) + .add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + const Q = k.loadOpts(a.query_ptr.addPtr(q_offset), .{ + .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded), + .other = initial_acc.to(cfg.dtype), + }); + const table = a.block_tables_ptr.addPtr(seq.to(.i64).mul(cfg.block_table_stride)); + const offsets = k.arange(0, cfg.tile_size, .i32); + var loop = k.openFor(0, partition.tiles_per_segment, 1, .{ initial_max, initial_sum, initial_acc }); + { + const selected = segment_start.add(loop.iv.mul(@as(i32, @intCast(cfg.tile_size)))).add(offsets); + const position = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = selected.lt(cfg.indices_count), + .other = k.full(&.{cfg.tile_size}, -1, .i32), + }); + const valid = selected.lt(cfg.indices_count).bitAnd(position.ge(0)).bitAnd(position.lt(seq_len)); + // A bounded single-tile segment was already checked above. + const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.constant(.i1, 1) else k.sum(valid.to(.i32)).gt(0); + var branch = k.openIfElse(nonempty, .{ k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{ cfg.block_m, cfg.head_size_padded }, .f32) }); + { + const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); + const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ + .mask = valid, + .other = k.zeros(&.{cfg.tile_size}, .i32), + }).to(.i64); + const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); + const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) + .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); + const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) + .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); + const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ + .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), + .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), + }); + const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ + .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), + .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), + }); + const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); + const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); + const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); + const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); + const alpha = k.exp2(loop.carried[0].sub(next_max)); + const P = k.exp2(S.sub(next_max.expandDims(1))); + const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); + const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); + branch.yieldThen(.{ next_max, sum, acc }); + } + { + branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); + } + loop.yield(.{ branch.results[0], branch.results[1], branch.results[2] }); + } + + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, loop.results[0], loop.results[1], loop.results[2]); + } + + fn storeResult(k: *Builder, cfg: Config, a: anytype, query: Value, head: Value, head_mask: Value, segment: Value, dims: Value, dim_mask: Value, max_score: Value, sum: Value, acc: Value) void { + if (split_k) { + // Adjacent heads are written together by the producer. + const stats_offset = query.to(.i64).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + k.storeOpts(a.segm_max_ptr.addPtr(stats_offset), max_score, .{ .mask = head_mask }); + k.storeOpts(a.segm_expsum_ptr.addPtr(stats_offset), sum, .{ .mask = head_mask }); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + // Zero-mass segments need no partial output; the reducer masks them. + k.storeOpts(a.output_ptr.addPtr(offset), acc, .{ .mask = k.mask2d(head_mask.bitAnd(sum.gt(0.0)), dim_mask, cfg.block_m, cfg.head_size_padded) }); + } else { + const denominator = k.where(sum.gt(0.0), sum, k.full(&.{cfg.block_m}, 1.0, .f32)); + const offset = query.to(.i64).mul(cfg.query_strides[0]).add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)).add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + k.storeOpts(a.output_ptr.addPtr(offset), acc.div(denominator.expandDims(1)).to(cfg.dtype), .{ .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded) }); + } + } + }; +} + +pub const SparseReducePtr = struct { + pub const Config = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + head_size: i64, + head_size_padded: i64, + head_tile: i64, + num_segments: i64, + launched_segments: i64, + output_strides: [3]i64, + }; + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_sparse_attention_ptr", + .inputs = &.{ "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "query_start_len_ptr" }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = try k.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + const query = k.programId(.x); + k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs)))); + const head = k.programId(.y); + const segments = k.arange(0, cfg.num_segments, .i32); + const segment_mask = segments.lt(cfg.launched_segments); + const dims = k.programId(.z).mul(@as(i32, @intCast(cfg.head_tile))).add(k.arange(0, cfg.head_tile, .i32)); + const dim_mask = dims.lt(cfg.head_size); + const stats = query.to(.i64).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + const maxima = k.loadOpts(a.segm_max_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32) }); + const sums = k.loadOpts(a.segm_expsum_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.zeros(&.{cfg.num_segments}, .f32) }); + const nonempty = segment_mask.bitAnd(sums.gt(0.0)); + const maximum = k.max(k.where(nonempty, maxima, k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32))); + const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.constant(.f32, 0.0)); + const rescale = k.exp2(maxima.sub(safe_maximum)); + const denominator = k.sum(sums.mul(rescale)); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + const partial = k.loadOpts(a.segm_output_ptr.addPtr(offset), .{ + .mask = k.mask2d(nonempty, dim_mask, cfg.num_segments, cfg.head_tile), + .other = k.zeros(&.{ cfg.num_segments, cfg.head_tile }, .f32), + }); + const numerator = k.sumOpts(partial.mul(rescale.expandDims(1)), .{ .axis = 0 }); + const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.constant(.f32, 1.0))); + const output_offset = query.to(.i64).mul(cfg.output_strides[0]).add(head.to(.i64).mul(cfg.output_strides[1])).add(dims.to(.i64).mul(cfg.output_strides[2])); + k.storeOpts(a.output_ptr.addPtr(output_offset), output.to(cfg.dtype), .{ .mask = dim_mask }); + } +}; diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig new file mode 100644 index 00000000..2a12e012 --- /dev/null +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -0,0 +1,861 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const triton = zml.kernel.triton; +const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernels = @import("unified_attention.zig"); +const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); + +const log = std.log.scoped(.@"zml/attention/triton"); + +fn isOneapiTarget() bool { + return zml.Compiler.current().platform.target == .oneapi; +} + +fn use2dKernel(all_decode: bool, batch_size: usize, num_kv_heads: usize) bool { + // Intel decode spills the 2D whole-sequence kernel; force the 3D split-K path. + if (all_decode and isOneapiTarget()) return false; + // prefill uses 2D; decode uses 3D until the batch is large enough to + // provide at least 128 2D launch programs across KV heads. + if (all_decode) { + const seq_threshold_3d = @divFloor(128, num_kv_heads); + return batch_size > seq_threshold_3d; + } + + return true; +} + +pub const Config2D = struct { + block_m: usize, + block_q: usize, + tile_size: usize, + num_warps: usize, + num_stages: usize, + total_q_blocks: usize, +}; + +fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { + const platform = zml.module.CompilationContext.current().platform; + return if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(expected) else false; +} + +/// Largest `BLOCK_M * HEAD_SIZE_PADDED` the 2D kernel survives. That product is +/// the shape of both the query tile and the f32 accumulator carried through the +/// tile loop; above it the kernel faults with CUDA_ERROR_MISALIGNED_ADDRESS. +/// +/// Measured on a GB300 with Gemma-4-31B, whose global-attention layers are +/// head_dim 512 (its sliding ones are 256): +/// +/// 64x256, 32x512, 16x512 ok 64x512, 128x512 fault +/// +/// The bound is on this product alone. Two things that do *not* move it, so do +/// not reach for them if a new head size faults: TILE_SIZE, which at 64 pushes +/// total shared memory to ~160 KB and still passes at BLOCK_M 32 while 64x512 +/// faults at 98 KB; and num_warps, where 8 halves the per-thread register share +/// and still faults -- which is also why the obvious "the accumulator outgrows +/// the register file" story cannot be the whole explanation. The underlying +/// Triton codegen bug is not diagnosed; this only keeps us out of it. +const max_query_tile_elements: usize = 16 * 1024; + +fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { + const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; + + // Until we test on other platforms, gate the fix to GB300 + const is_gb300 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); + + var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; + + var block_m = options.block_m; + var block_q = options.block_q; + if (options.max_seqlen_q >= 256) { + if (options.head_dim >= 256) { + block_m = 64; + tile_size = 16; + } else { + block_m = 128; + } + + if (is_gb300) { + // Halving BLOCK_M for every doubling of the head keeps that product + // constant: 128 at head_dim 128, 64 at 256, 32 at 512. The first two + // are the values this branch has always picked. + const head_dim_padded = std.math.ceilPowerOfTwoAssert(usize, options.head_dim); + block_m = @min(128, max_query_tile_elements / head_dim_padded); + } + num_stages_2d = 1; + num_warps = 4; + } + block_q = block_m / options.numQueriesPerKv(); + const total_q_blocks = options.num_tokens / block_q + options.batch_size; + + return .{ + .block_m = block_m, + .block_q = block_q, + .tile_size = tile_size, + .num_warps = num_warps, + .num_stages = @min(max_num_stages_2d, num_stages_2d), + .total_q_blocks = total_q_blocks, + }; +} + +pub const Config3D = struct { + const AttentionConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + total_q_blocks: usize, + }; + const ReduceConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + }; + attention: AttentionConfig, + reduce: ReduceConfig, +}; + +fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { + var reduce_num_warps: usize = 2; + // Intel decode needs more warps to spread the work and avoid register spill. + const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; + const tile_size = options.block_size; + + //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); + var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; + num_segments = std.math.ceilPowerOfTwoAssert(usize, num_segments); + num_segments = @min(num_segments, 128); + if (options.all_decode and !isOneapiTarget()) { + // Keep the number of segments small to then limit reduce cost + // Didn't change the computation of Intel decode at the momment + // Need to be tested + num_segments = @min(num_segments, 16); + } + const min_segments: usize = if (tile_size <= 16) 16 else 8; + num_segments = @max(num_segments, min_segments); + if (num_segments == min_segments) { + reduce_num_warps = 1; + } + + return .{ + .attention = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = attn_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + .total_q_blocks = options.total_q_blocks, + }, + .reduce = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = reduce_num_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + }, + }; +} + +pub fn getCuCount() usize { + const platform = zml.Compiler.current().platform; + if (platform.devices.len == 0) return 1; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); +} + +const SparseConfig = struct { + tile_size: usize, + num_segments: usize, +}; + +fn selectSparseConfig(count: usize, batch_size: usize, num_kv_heads: usize, cu_count: usize, all_decode: bool) SparseConfig { + std.debug.assert(count > 0 and batch_size > 0 and num_kv_heads > 0); + if (!all_decode) return .{ .tile_size = 32, .num_segments = 1 }; + + const target_num_prgms = cu_count * 4; + const occupancy_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, target_num_prgms * 4, batch_size * num_kv_heads) catch unreachable); + const tile: usize = if (occupancy_splits > 128) 16 else 32; + const tile_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, count, tile) catch unreachable); + + return .{ + .tile_size = tile, + .num_segments = @min(tile_splits, @min(if (tile == 16) @as(usize, 256) else 128, @max(8, occupancy_splits))), + }; +} + +pub const paged = struct { + pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } + }; + + pub const PagedAttentionOptions = struct { + cu_count: usize, + all_decode: bool, + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + batch_size: usize, + block_size: usize, + num_blocks: usize, + max_num_block_per_seq: usize, + sliding_window: usize, + block_m: usize, + block_q: usize, + total_q_blocks: usize, + target_num_prgms: usize, + num_2d_prgms: usize, + max_seqlen_q: usize, + scale: ?f32, + + pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { + return self.num_heads / self.num_kv_heads; + } + + pub fn maxSeqLenK(self: PagedAttentionOptions) usize { + return self.max_num_block_per_seq * self.block_size; + } + }; + + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) { + return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); + } + + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q.dim(.b)); + const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + + const sliding_window: usize = opts.mask.slidingWindow() orelse 0; + + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !parameters.options_.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q.dim(.hd)), + .batch_size = @intCast(parameters.block_table.dim(.b)), + .block_size = @intCast(k_cache.dim(.k_chunk)), + .num_blocks = @intCast(k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = parameters.options_.max_seqlen_q, + .scale = opts.scale, + }; + + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, + ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else + pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); + + return output; + } + + pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(opts.mask == .indices, "Expected indices mask but got {}", .{opts.mask}); + const indices = opts.mask.indices; + stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); + stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); + stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); + stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); + + const config = selectSparseConfig( + @intCast(opts.mask.indices.dim(.topk)), + @intCast(parameters.block_table.dim(.b)), + @intCast(q.dim(.hkv)), + getCuCount(), + !parameters.options_.is_prefill, + ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) + else + pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); + } + + pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments == 1); + + const indices = opts.mask.indices; + + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention2dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + 1, + }; + + const warps: i32 = if (all_decode and isOneapiTarget()) 8 else 4; + + const output = kernels_sparse.KernelSparseAttention2dPtr.Kernel.call(inputs, .{ + .output = q.shape(), + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + return output.output; + } + + pub fn pagedSparseAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments > 1); + + const indices = opts.mask.indices; + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention3dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const partition = kernels_sparse.SparseSegments.init(config.indices_count, config.num_segments, config.tile_size); + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + @intCast(partition.launched), + }; + + const warps: i32 = if (all_decode) 8 else 4; + + const stats_shape = zml.Shape.init(.{ q.dim(.b), config.num_segments, config.num_query_heads }, .f32); + + const partials = kernels_sparse.KernelSparseAttention3dPtr.Kernel.call(inputs, .{ + .output = .init(.{ q.dim(.b), config.num_query_heads, config.num_segments, head_size_padded }, .f32), + .segm_max = stats_shape, + .segm_expsum = stats_shape, + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + const head_tile = @min(64, head_size_padded); + + const output = kernels_sparse.SparseReducePtr.Kernel.call(.{ + .segm_output_ptr = partials.output, + .segm_max_ptr = partials.segm_max, + .segm_expsum_ptr = partials.segm_expsum, + .query_start_len_ptr = parameters.query_start_len, + }, .{ .output = q.shape() }, .{ + .cfg = .{ + .dtype = config.dtype, + .num_seqs = config.num_seqs, + .num_query_heads = config.num_query_heads, + .head_size = config.head_size, + .head_size_padded = head_size_padded, + .head_tile = head_tile, + .num_segments = config.num_segments, + .launched_segments = partition.launched, + .output_strides = config.query_strides, + }, + .grid = .{ @intCast(q.dim(.b)), @intCast(config.num_query_heads), @intCast(@divExact(head_size_padded, head_tile)) }, + .num_stages = 1, + .num_warps = 4, + }).output; + + return output; + } + + pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select2dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)), + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.block_q), + .block_m = @intCast(config.block_m), + .use_fp8 = false, + .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.mask.isCausal(), + }; + log.debug("pagedAttention2d config: {any}", .{kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .out_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ .output = q.shape() }, + .{ + .cfg = kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, + .num_stages = @intCast(config.num_stages), + .num_warps = @intCast(config.num_warps), + }, + ); + return output.output; + } + + pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select3dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.mask.isCausal(), + }; + log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); + + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const attn_grid: [3]i32 = .{ + @intCast(config.attention.total_q_blocks), + @intCast(paged_attention_opts.num_kv_heads), + @intCast(config.attention.num_segments_per_seq), + }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); + + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = .scalar(num_seqs, .i32), + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ + @intCast(paged_attention_opts.num_tokens), + @intCast(paged_attention_opts.num_heads), + 1, + }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); + + return output.output; + } + + /// oneAPI/Intel specialization of pagedAttention3d: routes the attention pass to + /// the SIMD16-tuned kernel (compile-time KV strides, cached loads, same-page fast + /// path) while reusing the shared segment-reduce kernel. See unified_attention_oneapi.zig. + pub fn pagedAttention3dOneapi(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select3dConfig(paged_attention_opts); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const attn_kernel_config: kernels_oneapi.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .is_causal = opts.mask.isCausal(), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, + .stride_k_cache_0 = k_strides[k_cache.axis(.page)], + .stride_k_cache_1 = k_strides[k_cache.axis(.k_chunk)], + .stride_k_cache_2 = k_strides[k_cache.axis(.hkv)], + .stride_v_cache_0 = v_strides[v_cache.axis(.page)], + .stride_v_cache_1 = v_strides[v_cache.axis(.k_chunk)], + .stride_v_cache_2 = v_strides[v_cache.axis(.hkv)], + // Intel: keep KV loads cached (.none); the streaming .cg hint lowers to + // fully-uncached here and defeats the L2 prefetcher. + .kv_cache_modifier = .none, + }; + log.debug("pagedAttention3dOneapi attention config: {any}", .{attn_kernel_config}); + + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .stride_k_cache_0_ptr = .scalar(k_strides[k_cache.axis(.page)], .i64), + .stride_k_cache_1_ptr = .scalar(k_strides[k_cache.axis(.k_chunk)], .i64), + .stride_k_cache_2_ptr = .scalar(k_strides[k_cache.axis(.hkv)], .i64), + .stride_v_cache_0_ptr = .scalar(v_strides[v_cache.axis(.page)], .i64), + .stride_v_cache_1_ptr = .scalar(v_strides[v_cache.axis(.k_chunk)], .i64), + .stride_v_cache_2_ptr = .scalar(v_strides[v_cache.axis(.hkv)], .i64), + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); + + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = .scalar(num_seqs, .i32), + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); + + return output.output; + } +}; diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig new file mode 100644 index 00000000..7b2b9428 --- /dev/null +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -0,0 +1,1121 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ttir = @import("mlir/dialects/ttir"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +pub const FP8_E4M3_MIN: f32 = -448.0; +pub const FP8_E4M3_MAX: f32 = 448.0; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +pub const KernelUnifiedAttention2dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + all_decode: bool = false, + is_causal: bool = true, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", + "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", + "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "query_start_len_ptr", + "num_seqs_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .out_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const out_scale = b.load(a.out_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention2d( + b, + a.output_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + out_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + output_stride_0, + output_stride_1, + qq_bias_stride_0, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `kernel_unified_attention_3d_ptr`. +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + is_causal: bool = true, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `reduce_segments_ptr`. +pub const ReduceSegmentsPtr = struct { + pub const Config = struct { + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + block_q: i64, + num_segments_per_seq: i64, + + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_segments_ptr", + .inputs = &.{ + "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "seq_lens_ptr", + "num_seqs_ptr", "out_scale_inv_ptr", "output_stride_0_ptr", "output_stride_1_ptr", + "block_table_stride_ptr", "query_start_len_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .out_scale_inv_ptr = .{ .ptr = .f32 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const num_seqs = b.load(a.num_seqs_ptr); + const out_scale_inv = b.load(a.out_scale_inv_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + + reduceSegments( + b, + a.output_ptr, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.seq_lens_ptr, + num_seqs, + out_scale_inv, + output_stride_0, + output_stride_1, + block_table_stride, + a.query_start_len_ptr, + cfg, + ); + } +}; + +// ============================================================================ +// Helpers +// ============================================================================ + +fn ctx() *mlir.Context { + return zml.Compiler.current().mlir_ctx; +} + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn fastExp(k: *Builder, x: Value) Value { + return k.exp2(x.mul(RCP_LN2)); +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +fn kernelUnifiedAttention2d( + k: *Builder, + output_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + out_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + output_stride_0: Value, + output_stride_1: Value, + qq_bias_stride_0: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention2dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); + + const kv_head_idx = k.programId(.x); + const q_block_global_idx = k.programId(.y); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + // Python: if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: return + const out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + // Q : (BLOCK_M, HEAD_SIZE_PADDED) + const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); + const q_mask_c = query_mask_1.expandDims(1); + const q_mask = q_mask_ab.bitAnd(q_mask_c); + const q_cache_mod: ttir.CacheModifier = + if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none; + const Q = k.loadOpts(query_ptr.addPtr(query_offset), .{ + .mask = q_mask, + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + .cache_modifier = q_cache_mod, + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init — sinks or -inf. + const m_init: Value = if (config.use_sinks) mb: { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + break :mb loaded.to(.f32).mul(RCP_LN2); + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // In the `use_qq_bias = true` branch, the tensor offset auto-splats the + // scalar ptr via addPtr. In the else branch there is no offset, so we + // need an explicit splatTo to give expandDims a tensor to work on. + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + // In causal mode, each query block only scans keys up to the last query + // represented in this block. In non-causal mode, scan the whole sequence + // so draft rows can attend to all rows written for the same sequence. + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; + + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + // Sliding-window tile pruning — comptime branch. + var tile_start: Value = k.constant(.i32, 0); + var tile_end: Value = num_tiles; + if (SLIDING_WINDOW > 0) { + const qpos_lo = q_local_bq; + if (config.is_causal) { + const qpos_hi_raw = qpos_lo.add(pad_term - 1); + const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + const last_allowed_key = context_len.add(qpos_hi); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); + } else { + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + } + } + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + const block_tables_ptr_shifted = block_tables_ptr.addPtr(block_table_offset); + + // Iterate through tiles (loop carries M, L, acc). + var loop = k.openFor(tile_start, tile_end, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + // k_offset : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + // K : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_mask = k.mask2d(dim_mask, tile_mask, HEAD_SIZE_PADDED, TILE_SIZE); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k_mask, + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + // V : (TILE_SIZE, HEAD_SIZE_PADDED) + const v_mask = k.mask2d(tile_mask, dim_mask, TILE_SIZE, HEAD_SIZE_PADDED); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = v_mask, + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // S : (BLOCK_M, TILE_SIZE) + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = ql_2d_i32.add(context_len).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + // S = tl.where(qmask_1 & qmask_0 & seq_mask, S, -inf) + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + // m_j = max(M, max(S, axis=1)) + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + // P = exp2(S - m_j[:, None]); l_j = sum(P, axis=1); alpha = exp2(M - m_j) + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + const new_M = m_j; + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ new_M, new_L, new_acc }); + } + const L_final = loop.results[1]; + var acc_final = loop.results[2]; + + const L_2d = k.expandDims(L_final, 1); + const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); + acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); + + if (config.use_fp8) { + acc_final = acc_final.mul(out_scale); + acc_final = k.clampf( + acc_final, + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_min, .f32), + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_max, .f32), + ); + } + + const oo0_2d = query_offset_0.expandDims(1).mul(output_stride_0); + const oo1_2d = query_offset_1.expandDims(1).mul(output_stride_1); + const output_offset = oo0_2d.add(oo1_2d).add(k.expandDims(offs_d, 0)); + + const store_mask = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + output_ptr.addPtr(output_offset), + acc_final.to(config.o_dtype), + .{ .mask = store_mask }, + ); +} + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} + +fn reduceSegments( + k: *Builder, + output_ptr: Value, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + seq_lens_ptr: Value, + num_seqs: Value, + out_scale_inv: Value, + output_stride_0: Value, + output_stride_1: Value, + block_table_stride: Value, + query_start_len_ptr: Value, + config: ReduceSegmentsPtr.Config, +) void { + _ = block_table_stride; + + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const BLOCK_Q: i64 = config.block_q; + + const query_token_idx = k.programId(.x); + const query_head_idx = k.programId(.y); + + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + + // The launch covers the padded query capacity, but the attention pass only + // writes scratch for active tokens. Zero the tail before looking up a + // sequence or dividing by its segment size (zero for inactive sequences). + const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); + const inactive = query_token_idx.ge(active_tokens); + k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ + .mask = dim_mask.bitAnd(inactive), + }); + k.returnIf(inactive); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); + + const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); + const segm_mask = seg_range.lt(act_num_segments); + + // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) + const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_offset = query_token_idx.to(.i64).mul(tok_stride) + .add(query_head_idx.mul(head_stride).to(.i64)) + .add(seg_range.to(.i64)); + + const segm_max = k.loadOpts(segm_max_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), + }); + const overall_max = k.max(segm_max); + + var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.zeros(&.{NUM_SEGMENTS_PER_SEQ}, .f32), + }); + segm_expsum = segm_expsum.mul(k.exp2(segm_max.sub(overall_max))); + const overall_expsum = k.sum(segm_expsum); + + // segm_output_offset : (NSPS, HSP) + const out_tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + const out_head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const head_size_padded_i32: i32 = @intCast(HEAD_SIZE_PADDED); + const base = query_token_idx.to(.i64).mul(out_tok_stride) + .add(query_head_idx.mul(out_head_stride).to(.i64)); + const seg_off_2d = b: { + const seg_col_i32 = k.expandDims(seg_range, 1); // (NSPS, 1) i32 + const seg_col_scaled = seg_col_i32.mul(head_size_padded_i32); // (NSPS, 1) i32 + break :b seg_col_scaled.to(.i64); // extsi → (NSPS, 1) i64 + }; + const dim_off_2d = b: { + const dim_row_i32 = k.expandDims(offs_d, 0); // (1, HSP) i32 + break :b dim_row_i32.to(.i64); // extsi → (1, HSP) i64 + }; + const seg_offset_2d_full = k.broadcastTo(base.add(seg_off_2d), &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const dim_offset_2d_full = k.broadcastTo(dim_off_2d, &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const segm_output_offset = seg_offset_2d_full.add(dim_offset_2d_full); + + var segm_output = k.loadOpts( + segm_output_ptr.addPtr(segm_output_offset), + .{ + .mask = k.mask2d(segm_mask, dim_mask, NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED), + .other = k.zeros(&.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }, .f32), + }, + ); + const rescale = k.exp2(segm_max.sub(overall_max)); + segm_output = segm_output.mul(rescale.expandDims(1)); + const acc_sum = k.sumOpts(segm_output, .{ .axis = 0 }); + + var acc = k.where( + overall_expsum.eq(0.0), + k.full(&.{HEAD_SIZE_PADDED}, 0.0, .f32), + acc_sum.div(overall_expsum), + ); + + if (config.use_fp8) { + acc = acc.mul(out_scale_inv); + acc = k.clampf( + acc, + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_min, .f32), + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_max, .f32), + ); + } + + k.storeOpts( + output_ptr.addPtr(output_offset), + acc.to(config.o_dtype), + .{ .mask = dim_mask }, + ); +} diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig new file mode 100644 index 00000000..b7c724f9 --- /dev/null +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -0,0 +1,536 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ttir = @import("mlir/dialects/ttir"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +// oneAPI/Intel (SIMD16) specialization of the 3D split-K paged-attention kernel. +// Duplicated from unified_attention.zig so the shared kernel stays backend-neutral; +// only the Intel decode path routes here (see triton_attention.zig pagedAttention3dOneapi). + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + is_causal: bool = true, + + stride_k_cache_0: i64 = 0, + stride_k_cache_1: i64 = 0, + stride_k_cache_2: i64 = 0, + stride_k_cache_3: i64 = 1, + stride_v_cache_0: i64 = 0, + stride_v_cache_1: i64 = 0, + stride_v_cache_2: i64 = 0, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + kv_cache_modifier: ttir.CacheModifier = .none, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr_oneapi", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const stride_k_cache_0 = b.constant(.i64, cfg.stride_k_cache_0); + const stride_k_cache_1 = b.constant(.i64, cfg.stride_k_cache_1); + const stride_k_cache_2 = b.constant(.i64, cfg.stride_k_cache_2); + const stride_v_cache_0 = b.constant(.i64, cfg.stride_v_cache_0); + const stride_v_cache_1 = b.constant(.i64, cfg.stride_v_cache_1); + const stride_v_cache_2 = b.constant(.i64, cfg.stride_v_cache_2); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_k_cache_2: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + stride_v_cache_2: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + const max_seq_prefix_len = if (config.is_causal) max_prefix_raw.minimum(seq_len) else seq_len; + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = config.kv_cache_modifier; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const same_block = TILE_SIZE == BLOCK_SIZE; + const in_block_offs: Value = if (same_block) offs_t else seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))); + const physical_block_idx: Value = if (same_block) + k.load(block_tables_ptr.addPtr(block_table_offset).addPtr(j)).to(.i64).splatTo(&.{TILE_SIZE}) + else + k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(in_block_offs, 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(in_block_offs, 0).mul(stride_k_cache_1)); + + const block_io_loads = same_block and HEAD_SIZE_PADDED == HEAD_SIZE; + + const K_load = if (block_io_loads) + k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ .cache_modifier = kv_cache_mod }) + else kb: { + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + break :kb k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const V_raw = if (block_io_loads) + k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ .cache_modifier = kv_cache_mod }) + else vb: { + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + break :vb k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const V_load = if (block_io_loads) + k.where( + k.broadcastTo(k.expandDims(tile_mask, 1), &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + V_raw, + k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + ) + else + V_raw; + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} diff --git a/zml/logo.zig b/zml/logo.zig new file mode 100644 index 00000000..235bc43d --- /dev/null +++ b/zml/logo.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +pub const Color = enum { + reset, + cyan, + sky, + blue, + violet, + pink, + shine_cyan, + shine_sky, + shine_blue, + shine_violet, + shine_pink, + + pub fn escape(self: Color, support: ColorSupport) []const u8 { + return switch (support) { + .true_color => self.trueColor(), + ._256 => self.xterm256(), + else => "", + }; + } + + fn trueColor(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;2;0;255;255m", + .sky => "\x1b[38;2;0;221;255m", + .blue => "\x1b[38;2;135;143;255m", + .violet => "\x1b[38;2;215;90;219m", + .pink => "\x1b[38;2;242;48;174m", + .shine_cyan => "\x1b[1;38;2;140;255;255m", + .shine_sky => "\x1b[1;38;2;128;244;255m", + .shine_blue => "\x1b[1;38;2;205;210;255m", + .shine_violet => "\x1b[1;38;2;255;170;255m", + .shine_pink => "\x1b[1;38;2;255;150;220m", + }; + } + + fn xterm256(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;5;51m", + .sky => "\x1b[38;5;45m", + .blue => "\x1b[38;5;105m", + .violet => "\x1b[38;5;170m", + .pink => "\x1b[38;5;205m", + .shine_cyan => "\x1b[1;38;5;123m", + .shine_sky => "\x1b[1;38;5;123m", + .shine_blue => "\x1b[1;38;5;189m", + .shine_violet => "\x1b[1;38;5;219m", + .shine_pink => "\x1b[1;38;5;212m", + }; + } +}; + +pub const Row = struct { + text: []const u8, + color: Color = .reset, + shine: Color = .reset, +}; + +pub const Block = struct { + rows: []const Row, +}; + +pub const ColorSupport = enum { + no_color, + ansi, + _256, + true_color, +}; + +pub const Options = struct { + color_support: ColorSupport, + is_shiny: *const fn ([]const u8) bool = isShineGlyph, +}; + +pub const zml_art_blocks = [_]Block{ + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ███████╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ╚══███╔╝", .color = .sky, .shine = .shine_sky }, + .{ .text = " ███╔╝ ", .color = .blue, .shine = .shine_blue }, + .{ .text = " ███╔╝ ", .color = .violet, .shine = .shine_violet }, + .{ .text = " ███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = " ╚══════╝", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "███╗ ███╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "████╗ ████║", .color = .sky, .shine = .shine_sky }, + .{ .text = "██╔████╔██║", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║╚██╔╝██║", .color = .violet, .shine = .shine_violet }, + .{ .text = "██║ ╚═╝ ██║", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚═╝ ╚═╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = "██╗ ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "██║ ", .color = .sky, .shine = .shine_sky }, + .{ .text = "██║ ", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║ .ai", .color = .violet, .shine = .shine_violet }, + .{ .text = "███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚══════╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, +}; + +pub fn detectColorSupport(io: std.Io, file: std.Io.File, env: *const std.process.Environ.Map) ColorSupport { + var no_color: bool = false; + var clicolor_force: bool = false; + no_color = if (env.get("NO_COLOR")) |v| v.len != 0 else false; + clicolor_force = if (env.get("CLICOLOR_FORCE")) |v| v.len != 0 else false; + + const mode = std.Io.Terminal.Mode.detect(io, file, no_color, clicolor_force) catch return .no_color; + if (mode != .escape_codes) return .no_color; + + if (env.get("COLORTERM")) |value| { + if (std.ascii.eqlIgnoreCase(value, "truecolor") or + std.ascii.eqlIgnoreCase(value, "24bit")) + { + return .true_color; + } + } + + if (env.get("TERM")) |value| { + if (std.mem.endsWith(u8, value, "-direct")) return .true_color; + if (std.mem.indexOf(u8, value, "truecolor") != null) return .true_color; + + if (std.mem.indexOf(u8, value, "256color") != null) return ._256; + } + + return .ansi; +} + +pub fn writeZmlArt(writer: *std.Io.Writer, options: Options) !void { + try write(writer, zml_art_blocks[0..], options); +} + +pub fn write(writer: *std.Io.Writer, blocks: []const Block, options: Options) !void { + if (blocks.len == 0) return; + + const row_count = blocks[0].rows.len; + for (blocks[1..]) |block| { + if (block.rows.len != row_count) return error.InvalidLogoBlocks; + } + + for (0..row_count) |row_index| { + const last_block = findLastVisibleBlockIndex(blocks, row_index) orelse { + try writer.writeAll("\n"); + continue; + }; + + for (blocks[0 .. last_block + 1], 0..) |block, block_index| { + const row = block.rows[row_index]; + try writeRow(writer, row, options, block_index == last_block); + } + try writer.writeAll(Color.reset.escape(options.color_support)); + try writer.writeAll("\n"); + } +} + +pub fn writeRow(writer: *std.Io.Writer, row: Row, options: Options, rtrim_whitespace: bool) !void { + const text = if (rtrim_whitespace) trimRightWhitespace(row.text) else row.text; + if (text.len == 0) return; + + const color = row.color.escape(options.color_support); + const shine = row.shine.escape(options.color_support); + + var view: std.unicode.Utf8View = .initUnchecked(text); + var iter = view.iterator(); + var is_shiny = false; + + try writer.writeAll(color); + while (iter.nextCodepointSlice()) |codepoint| { + const should_shine = options.is_shiny(codepoint); + if (should_shine != is_shiny) { + try writer.writeAll(if (should_shine) shine else color); + is_shiny = should_shine; + } + try writer.writeAll(codepoint); + } +} + +fn findLastVisibleBlockIndex(blocks: []const Block, row_index: usize) ?usize { + var index = blocks.len; + while (index > 0) { + index -= 1; + if (trimRightWhitespace(blocks[index].rows[row_index].text).len != 0) return index; + } + return null; +} + +fn trimRightWhitespace(text: []const u8) []const u8 { + return std.mem.trimEnd(u8, text, " \t\r\n"); +} + +pub fn isShineGlyph(codepoint: []const u8) bool { + return std.mem.eql(u8, codepoint, "╔") or + std.mem.eql(u8, codepoint, "╗") or + std.mem.eql(u8, codepoint, "╚") or + std.mem.eql(u8, codepoint, "╝") or + std.mem.eql(u8, codepoint, "═") or + std.mem.eql(u8, codepoint, "║") or + std.mem.eql(u8, codepoint, ".") or + std.mem.eql(u8, codepoint, "a") or + std.mem.eql(u8, codepoint, "i"); +} diff --git a/zml/mem.zig b/zml/mem.zig index c12a2f61..a64ae968 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,8 +22,8 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, - .rocm, .cpu => .{ .passthrough = parent }, + .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, + .cpu, .neuron, .metal => .{ .passthrough = parent }, }; } @@ -225,9 +225,7 @@ pub const FixedBufferPool = struct { pub const DynamicBufferPool = struct { const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .max(.of(u8), .of(Node)); - const ItemPtr = [*]align(alignment.toByteUnits()) u8; - const Slice = []align(alignment.toByteUnits()) u8; + const alignment: std.mem.Alignment = .of(Node); block_size: usize, max_blocks: usize, @@ -311,8 +309,8 @@ pub const DynamicBufferPool = struct { } } - fn nodeToSlice(self: *DynamicBufferPool, node: *Node) Slice { - const ptr: ItemPtr = @ptrCast(@alignCast(node)); + fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { + const ptr: [*]u8 = @ptrCast(node); return ptr[0..self.block_size]; } }; @@ -331,14 +329,20 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const ModelBufferized = Bufferized(Model); if (ModelBufferized == Buffer) { + bufferized_.* = .{ + // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. + ._shape = model._shape, + ._shards = .empty, + ._platform = undefined, + }; return; } const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.fields) |field| { - try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); + inline for (struct_type_info.field_names) |field_name| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); } }, .@"union" => { @@ -360,7 +364,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index 1274761b..84fb9d36 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -20,27 +20,30 @@ pub fn MapType(From: type, To: type) type { return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var same: bool = true; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - if (!field.is_comptime) { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for ( + struct_field_names[0..], + struct_field_types[0..], + struct_field_attrs[0..], + struct_infos.field_names, + struct_infos.field_types, + struct_infos.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + if (!source_attrs.@"comptime") { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ - .@"comptime" = field.is_comptime, + .@"comptime" = source_attrs.@"comptime", .@"align" = @alignOf(R), }; same = false; @@ -52,34 +55,35 @@ pub fn MapType(From: type, To: type) type { } } } else { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } } if (same) return T; return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { - const fields = union_info.fields; + const field_names = union_info.field_names; var same: bool = true; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { - const R = map(field.type); - if (R == field.type) { - field_name.* = field.name; - field_type.* = field.type; - field_attr.* = .{ - .@"align" = field.alignment, - }; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for ( + union_field_names[0..], + union_field_types[0..], + union_field_attrs[0..], + union_info.field_names, + union_info.field_types, + union_info.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - field_name.* = field.name; + field_name.* = source_name; field_type.* = R; field_attr.* = .{ .@"align" = @alignOf(R), @@ -92,11 +96,11 @@ pub fn MapType(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), @@ -139,7 +143,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -148,7 +152,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam // TODO: handle tuple to slice conversion // TODO: handle error bubbling up const From = FnParam(cb, 1); - const To = stdx.meta.FnResult(cb); + const To = stdx.meta.FnReturn(cb); const FromStruct = @TypeOf(from); const type_info_to_ptr = @typeInfo(@TypeOf(to)); if (type_info_to_ptr != .pointer) { @@ -176,29 +180,34 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } + if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { + to.* = from; + return; + } + if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { - .@"struct" => |info| inline for (info.fields) |field| { - if (field.is_comptime or @sizeOf(field.type) == 0) continue; - const field_type_info = @typeInfo(field.type); + .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; + const field_type_info = @typeInfo(Field); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field.name), - &@field(to, field.name), + @field(from, field_name), + &@field(to, field_name), ); - } else if (field.default_value_ptr) |_| { - @field(to, field.name) = null; + } else if (field_attrs.default_value_ptr) |_| { + @field(to, field_name) = null; } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); }, - else => @field(to, field.name) = @field(from, field.name), + else => @field(to, field_name) = @field(from, field_name), } }, .@"union" => { @@ -222,7 +231,11 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), + else => { + const item = try allocator.create(ptr_info.child); + try mapAlloc(cb, allocator, ctx, from.*, item); + to.* = item; + }, }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -231,6 +244,10 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, + .many => if (FromStruct == [*:0]const u8 and ToStruct == FromStruct and From != u8) { + // Tensor axis tags are borrowed immutable strings. + to.* = from; + } else stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), }, .optional => if (from) |f| { @@ -259,7 +276,9 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, + pointer: *const A, other: u8, + tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. comptime static_val: u8 = 8, comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, @@ -269,7 +288,9 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, + pointer: *const B, other: u8, + tag: [*:0]const u8, static_val: u8, static_slice: []B, field_with_empty: struct { B, Empty }, @@ -279,7 +300,9 @@ test mapAlloc { .field = .{ .a = 4 }, .array = .{ .{ .a = 5 }, .{ .a = 6 } }, .other = 7, + .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -287,16 +310,19 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); + defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); try testing.expectEqual(6, bb.array[1].b); try testing.expectEqual(7, bb.other); + try testing.expectEqual(aa.tag, bb.tag); try testing.expectEqual(8, bb.static_val); try testing.expectEqual(9, bb.slice[0].b); try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); + try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: @@ -324,25 +350,21 @@ pub fn MapRestrict(From: type, To: type) type { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. // We map each field individually. Fields without From and comptime fields are removed. - const fields = struct_infos.fields; + const field_names = struct_infos.field_names; var num_fields: usize = 0; - var struct_field_names: [fields.len][]const u8 = undefined; - var struct_field_types: [fields.len]type = undefined; - var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; - for (fields) |field| { - if (!field.is_comptime and Contains(field.type, From)) { - const R = map(field.type); - if (R == field.type) { - struct_field_names[num_fields] = field.name; - struct_field_types[num_fields] = field.type; - struct_field_attrs[num_fields] = .{ - .@"comptime" = field.is_comptime, - .@"align" = field.alignment, - .default_value_ptr = field.default_value_ptr, - }; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, From)) { + const R = map(Field); + if (R == Field) { + struct_field_names[num_fields] = field_name; + struct_field_types[num_fields] = Field; + struct_field_attrs[num_fields] = field_attrs; } else { - const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; + const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; struct_field_names[num_fields] = name; struct_field_types[num_fields] = R; struct_field_attrs[num_fields] = .{ @@ -369,13 +391,13 @@ pub fn MapRestrict(From: type, To: type) type { .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". - const fields = union_info.fields; - var union_field_names: [fields.len][]const u8 = undefined; - var union_field_types: [fields.len]type = undefined; - var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; - for (0.., fields) |i, field| { - const FT = map(field.type); - union_field_names[i] = field.name; + const field_names = union_info.field_names; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { + const FT = map(Field); + union_field_names[i] = field_name; union_field_types[i] = FT; union_field_attrs[i] = .{ .@"align" = @alignOf(FT), @@ -385,19 +407,19 @@ pub fn MapRestrict(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.is_const) + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), - .many => if (ptr_info.is_const) + .many => if (ptr_info.attrs.@"const") [*]const map(ptr_info.child) else [*]map(ptr_info.child), - .c => if (ptr_info.is_const) + .c => if (ptr_info.attrs.@"const") [*c]map(ptr_info.child) else [*c]map(ptr_info.child), @@ -438,90 +460,98 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. pub fn VisitReturn(comptime cb: anytype) type { - return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| + return if (stdx.meta.FnReturnErrorSet(cb)) |error_set| error_set!void else void; } -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { - const Callback = @TypeOf(cb); - const Ptr = @TypeOf(v); - const type_info_v = @typeInfo(Ptr); - if (type_info_v != .pointer) { - stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); - } - const ptr_info = type_info_v.pointer; - const Child = ptr_info.child; - const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; - - const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { - .pointer => |info| .{ info.child, !info.is_const }, - else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), +const Visitor = struct { + pub const Action = enum { + callback, + recurse, + exit, }; - // Abort if v doesnt' contain any K. - if (comptime !Contains(Ptr, K)) return; - // Handle simple cases. - switch (Ptr) { - *const K, *K => return cb(ctx, v), - *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, - []const K, []K => { - for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); - return; - }, - else => {}, - } + pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { + const ptr_info = switch (@typeInfo(PtrTypeOfV)) { + .pointer => |info| info, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a pointer/slice input, but received: {any}", .{ @TypeOf(callback), PtrTypeOfV }), + }; - // Handle stdx.BoundedArray that contains uninitalized data. - if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { - return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); - } + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; - // Recursively visit fields of v. - switch (ptr_info.size) { - .one => switch (@typeInfo(Child)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - switch (field_type_info) { - .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), - .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), - else => {}, - } + if (ptr_info.size == .one and ptr_info.child == TargetType) { + return .callback; + } + + switch (@typeInfo(ptr_info.child)) { + .@"struct", .@"union" => { + if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; }, - .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), - }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), - }, - .slice => { - for (v) |*v_elem| { - switch (@typeInfo(Child)) { - .@"struct" => |s| inline for (s.fields) |field| { - if (field.is_comptime or comptime !Contains(field.type, K)) continue; - const field_type_info = @typeInfo(field.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - if (field_type_info == .pointer) { - if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); + else => {}, + } + + return .recurse; + } +}; + +pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + + return switch (action) { + .exit => {}, + .callback => callback(ctx, v), + .recurse => { + const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| .{ info.child, !info.attrs.@"const" }, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + if (comptime !Contains(PtrTypeOfV, TargetType)) return; + + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return visit(callback, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + if (@typeInfo(Field) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); } else { - if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); + if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); } }, - .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), - .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), - .@"union" => switch (v_elem.*) { - inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), + .array => for (v[0..]) |*elem| { + if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem); }, - else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), - } + .optional => return if (v.*) |*opt| visit(callback, ctx, opt), + .@"union" => return switch (v.*) { + inline else => |*v_field| visit(callback, ctx, v_field), + }, + .pointer => return visit(callback, ctx, v.*), + else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), + }, + .slice => { + for (v) |*v_elem| { + if (can_error) try visit(callback, ctx, v_elem) else visit(callback, ctx, v_elem); + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit({any}) doesn't support [*] style pointers got {any}", .{ @TypeOf(callback), ChildTypeV }), } }, - .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), - } + }; } test visit { @@ -530,20 +560,26 @@ test visit { const NestedAttr = struct { nested: Attr }; const NestedAttrOptional = struct { nested: ?Attr }; const SimpleStruct = struct { prop: Attr }; - const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr, prop4: []Attr }; const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; - const LocalContext = struct { result: usize }; + const LocalContext = struct { + result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } + }; { var context: LocalContext = .{ .result = 0 }; const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -551,37 +587,28 @@ test visit { var context: LocalContext = .{ .result = 0 }; var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.mutCb, &context, &container); try std.testing.expectEqual(1, context.result); } { var context: LocalContext = .{ .result = 0 }; - var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null, .prop4 = &more_props }; - try std.testing.expectEqual(1, context.result); + visit(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1 + 4 + 5 + 6, context.result); } { var context: LocalContext = .{ .result = 0 }; - const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 }, .prop4 = &more_props }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); - try std.testing.expectEqual(3, context.result); + try std.testing.expectEqual(1 + 2 + 4 + 5 + 6, context.result); } { var context: LocalContext = .{ .result = 0 }; @@ -597,16 +624,144 @@ test visit { .prop5 = prop5, // 4 will be counted twice. }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); try std.testing.expectEqual(14, context.result); } } +/// Similar to `visit` but doesn't follow pointers or slices. +/// This guarantees that the given type is a flat struct. +/// +/// see: `zml.meta.visit` +pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visitStruct({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + const err_msg = "zml.meta.visitStruct only handles flat struct, found pointer in {}"; + const err_args = .{ChildTypeV}; + + switch (action) { + .exit => {}, + .callback => return callback(ctx, v), + .recurse => { + if (comptime !Contains(ChildTypeV, TargetType)) return; + + // bounded array support (like visit) + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return for (v.slice()) |*elem| { + if (can_error) + try visitFlatStruct(callback, ctx, elem) + else + visitFlatStruct(callback, ctx, elem); + }; + } + + return switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); + + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); + }, + .array => for (v[0..]) |*elem| { + if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); + }, + .optional => if (v.*) |*opt| visitFlatStruct(callback, ctx, opt), + .@"union" => switch (v.*) { + inline else => |*v_field| visitFlatStruct(callback, ctx, v_field), + }, + else => stdx.debug.compileError(err_msg, err_args), + }, + else => stdx.debug.compileError(err_msg, err_args), + }; + }, + } +} + +test visitFlatStruct { + const Attr = struct { data: usize }; + const OtherAttr = struct { other: []const u8 }; + const NestedAttr = struct { nested: Attr }; + const NestedAttrOptional = struct { nested: ?Attr }; + const SimpleStruct = struct { prop: Attr }; + // Note: no slice anymore, no stdx.BoundedArray + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; + + const LocalContext = struct { + result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } + }; + + { + var context: LocalContext = .{ .result = 0 }; + const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; + + visitFlatStruct(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1 + 2, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; + const container: NestedTypesStruct = .{ + .prop1 = .{ .data = 1 }, + .prop2 = .{ .other = "hello" }, + .prop3 = .{ .nested = .{ .data = 2 } }, + .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. + }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1 + 2 + 3 + 4 + 4, context.result); + } +} pub fn count(T: type, value: anytype) u32 { var counter: u32 = 0; visit(struct { @@ -645,17 +800,17 @@ pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.fields) |f| { - if (f.is_comptime) continue; - if (@typeInfo(f.type) == .pointer) { + inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime") continue; + if (@typeInfo(Field) == .pointer) { stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(f.type, values.len); + var fields = try allocator.alloc(Field, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, f.name); + fields[i] = @field(val, field_name); } - @field(out, f.name) = try zip(func, allocator, fields, args); + @field(out, field_name) = try zip(func, allocator, fields, args); } return out; }, @@ -702,13 +857,13 @@ pub fn collectAlloc( func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, obj: anytype, -) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); +) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, - out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, + out: std.ArrayList(stdx.meta.FnReturn(func)) = .empty, oom: bool = false, fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { @@ -729,8 +884,8 @@ pub fn collectAlloc( /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. -pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), out: @TypeOf(out), @@ -749,15 +904,79 @@ pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: std.debug.assert(context.idx == context.out.len); } +/// Finds all X in the given object, and write their pointers into an arraylist. +pub fn collectPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*const T { + const CollectPtrCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(*const T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val) catch { + ctx.oom = true; + }; + } + }; + var context = CollectPtrCtx{ .allocator = allocator }; + visit(CollectPtrCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + +/// Finds all X in the given object, and write their mutable pointers into an arraylist. +pub fn collectMutPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*T { + const ok = switch (@typeInfo(@TypeOf(obj))) { + .pointer => |ptr_info| !ptr_info.attrs.@"const", + else => false, + }; + stdx.debug.assertComptime(ok, "zml.meta.collectMutPtrs requires a mutable pointer input, got: {}", .{@TypeOf(obj)}); + return @ptrCast(try collectPtrs(T, allocator, obj)); +} + +/// Finds all X in the given object, and copy them by value into an arraylist. +pub fn collectValues( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]T { + const CollectValuesCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val.*) catch { + ctx.oom = true; + }; + } + }; + var context = CollectValuesCtx{ .allocator = allocator }; + visit(CollectValuesCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (params.len == 1) return void; - return params[0].type orelse @compileError("anytype not supported in collect"); + return params[0] orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".params; - return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + return params[params.len - 1] orelse @compileError("anytype not supported in collect"); } pub fn Contains(Haystack: type, T: type) bool { @@ -772,15 +991,18 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { - inline for (info.fields) |field| { - if (!field.is_comptime and Contains(field.type, T)) + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; + inline for (info.field_types, info.field_attrs) |Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, T)) { return true; + } } return false; }, .@"union" => |info| { - inline for (info.fields) |field| { - if (Contains(field.type, T)) + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; + inline for (info.field_types) |Field| { + if (Contains(Field, T)) return true; } return false; diff --git a/zml/mlirx.zig b/zml/mlirx.zig index 748d988c..bc92eee0 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -8,80 +8,40 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); pub const Type = struct { + pub fn rankedTensor(ctx: *mlir.Context, shape: Shape) *const mlir.Type { + return .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx, shape.dtype())); + } + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => mlir.integerType(ctx, .i1), - .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), - .f8e3m4 => mlir.floatType(ctx, .f8e3m4), - .f8e4m3 => mlir.floatType(ctx, .f8e4m3), - .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), - .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), - .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), - .f8e5m2 => mlir.floatType(ctx, .f8e5m2), - .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), - .f8e8m0 => @panic("Unsupported type"), - .bf16 => mlir.floatType(ctx, .bf16), - .f16 => mlir.floatType(ctx, .f16), - .f32 => mlir.floatType(ctx, .f32), - .f64 => mlir.floatType(ctx, .f64), - .i2 => mlir.integerType(ctx, .i2), - .i4 => mlir.integerType(ctx, .i4), - .i8 => mlir.integerType(ctx, .i8), - .i16 => mlir.integerType(ctx, .i16), - .i32 => mlir.integerType(ctx, .i32), - .i64 => mlir.integerType(ctx, .i64), - .u2 => mlir.integerType(ctx, .u2), - .u4 => mlir.integerType(ctx, .u4), - .u8 => mlir.integerType(ctx, .u8), - .u16 => mlir.integerType(ctx, .u16), - .u32 => mlir.integerType(ctx, .u32), - .u64 => mlir.integerType(ctx, .u64), - .c64 => mlir.complexType(ctx, .c64), - .c128 => mlir.complexType(ctx, .c128), + .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + .f8e8m0 => .float(ctx, .f8e8m0fnu), + .bf16 => .float(ctx, .bf16), + .f16 => .float(ctx, .f16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), + .u4 => .int(ctx, .u4), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .c64 => .complex(ctx, .c64), + .c128 => .complex(ctx, .c128), }; } - - // TODO(Corentin): Maybe remove that as it now requires a mlir context. - // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. - pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { - const mapping = .{ - .{ .bool, mlir.integerType(ctx, .i1) }, - - .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, - .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, - .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, - .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, - .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, - .{ .bf16, mlir.floatType(ctx, .bf16) }, - .{ .f16, mlir.floatType(ctx, .f16) }, - .{ .f32, mlir.floatType(ctx, .f32) }, - .{ .f64, mlir.floatType(ctx, .f64) }, - - .{ .i4, mlir.integerType(ctx, .i4) }, - .{ .i8, mlir.integerType(ctx, .i8) }, - .{ .i16, mlir.integerType(ctx, .i16) }, - .{ .i32, mlir.integerType(ctx, .i32) }, - .{ .i64, mlir.integerType(ctx, .i64) }, - - .{ .u4, mlir.integerType(ctx, .u4) }, - .{ .u8, mlir.integerType(ctx, .u8) }, - .{ .u16, mlir.integerType(ctx, .u16) }, - .{ .u32, mlir.integerType(ctx, .u32) }, - .{ .u64, mlir.integerType(ctx, .u64) }, - - .{ .c64, mlir.complexType(ctx, .c64) }, - .{ .c128, mlir.complexType(ctx, .c128) }, - }; - - inline for (mapping) |entry| { - const dt, const mlirT = entry; - if (mlirT.eql(mlir_type)) { - return dt; - } - } - - std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); - } }; diff --git a/zml/module.zig b/zml/module.zig deleted file mode 100644 index 31637852..00000000 --- a/zml/module.zig +++ /dev/null @@ -1,536 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const dialects = @import("mlir/dialects"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); -const upb = @import("upb"); - -const Buffer = @import("buffer.zig").Buffer; -const Exe = @import("exe.zig").Exe; -const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/module"); - -const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - -pub const CompilationContext = struct { - pub const Scope = struct { - block: *mlir.Block, - id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), - id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), - arena: std.heap.ArenaAllocator, - - pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { - const arena: std.heap.ArenaAllocator = .init(allocator); - return .{ - .block = block, - .id_to_argument = .empty, - .id_to_donation = .empty, - .id_to_output_memory_kind = .empty, - .arena = arena, - }; - } - - pub fn deinit(self: *Scope) void { - self.arena.deinit(); - } - }; - - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - - mlir_registry: *mlir.DialectRegistry, - mlir_ctx: *mlir.Context, - mlir_pass_manager: *mlir.PassManager, - //mlir_op_pass_manager: *mlir.OpPassManager, - module: *mlir.Module, - platform: *const Platform, - - scopes: stdx.BoundedArray(Scope, 16) = .{}, - - threadlocal var _current: ?*CompilationContext = null; - - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) CompilationContext { - _ = io; // autofix - mlir.registerPasses("Transforms"); - const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - inline for (.{ "func", "stablehlo" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); - } - var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); - const module = mlir.Module.init(.unknown(mlir_ctx)); - module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); - - const pass_manager = mlir.PassManager.init(mlir_ctx); - { - var opm = pass_manager.asOpPassManager(); - const passes: []const []const u8 = &.{ - "canonicalize", - "cse", - "canonicalize", - }; - for (passes) |pass| { - opm.addPipeline(pass) catch unreachable; - } - } - - return .{ - .allocator = allocator, - .arena = std.heap.ArenaAllocator.init(allocator), - .mlir_registry = mlir_registry, - .mlir_ctx = mlir_ctx, - .mlir_pass_manager = pass_manager, - .module = module, - .platform = platform, - }; - } - - pub fn deinit(self: *CompilationContext) void { - self.mlir_pass_manager.deinit(); - self.module.deinit(); - self.mlir_ctx.deinit(); - self.mlir_registry.deinit(); - self.arena.deinit(); - } - - pub fn activate(self: *CompilationContext) void { - std.debug.assert(_current == null); - _current = self; - } - - pub fn deactivate(self: *CompilationContext) void { - _ = self; - _current = null; - } - - pub fn current() *CompilationContext { - return _current.?; - } - - pub fn currentScope(self: *CompilationContext) *Scope { - return &self.scopes.slice()[self.scopes.len - 1]; - } - - pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { - const scope = Scope.initFromBlock(self.allocator, block); - self.scopes.appendAssumeCapacity(scope); - } - - pub fn popBlock(self: *CompilationContext) void { - var maybe_popped_scope = self.scopes.pop(); - if (maybe_popped_scope) |*popped| { - popped.deinit(); - } - } - - //pub fn addShardingAttributes( - // self: *const CompilationContext, - // mesh: Mesh, - // arg_attrs: []AttributeList, - // res_attrs: []AttributeList, - // input_shapes: []const Shape, - // input_meshes: []const ?Mesh, - // output_info: OutputInfo, - //) void { - // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. - - // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); - // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { - // attr.appendAssumeCapacity(default_layout); - // if (input_mesh) |m| { - // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); - // } else { - // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - // } - - // for (res_attrs, output_info.shapes) |*attr, shape| { - // attr.appendAssumeCapacity(default_layout); - // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); - // } - //} - - //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { - // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; - // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; - // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); - //} -}; - -pub fn compile( - allocator: std.mem.Allocator, - io: std.Io, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), - platform: *const Platform, -) !Exe { - var compilation_context: CompilationContext = .init(allocator, io, platform); - defer compilation_context.deinit(); - - const result = emitMlir(&compilation_context, func, args) catch unreachable; - defer result.output_info.deinit(compilation_context.allocator); - defer result.input_info.deinit(compilation_context.allocator); - - _ = result.func.appendTo(compilation_context.module.body()); - const sharding = platform.sharding(); - compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); - compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; - - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); - - const num_devices = sharding.num_partitions * sharding.num_replicas; - const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); - errdefer exe.deinit(); - - return exe; -} - -fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { - const LocalContext = struct { - list: *std.array_list.Managed(Shape), - }; - var list = std.array_list.Managed(Shape).init(allocator); - errdefer list.deinit(); - - var context: LocalContext = .{ .list = &list }; - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.list.append(tensor.shape()); - } - }.cb, &context, v); - - return try list.toOwnedSlice(); -} - -pub const OutputInfo = struct { - shapes: []Shape, - values: []*const mlir.Value, - donations: []?usize, - output_memory_kinds: []Memory.Kind, - - pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - allocator.free(self.values); - allocator.free(self.donations); - allocator.free(self.output_memory_kinds); - } -}; - -fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - value_list: *std.array_list.Managed(*const mlir.Value), - donation_list: *std.array_list.Managed(?usize), - output_memory_kind_list: *std.array_list.Managed(Memory.Kind), - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); - errdefer value_list.deinit(); - var donation_list = std.array_list.Managed(?usize).init(allocator); - errdefer donation_list.deinit(); - var output_memory_kind_list = std.array_list.Managed(Memory.Kind).init(allocator); - errdefer output_memory_kind_list.deinit(); - - var context: LocalContext = .{ - .shape_list = &shape_list, - .value_list = &value_list, - .donation_list = &donation_list, - .output_memory_kind_list = &output_memory_kind_list, - }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - try ctx_.value_list.append(tensor.value()); - try ctx_.donation_list.append(tensor.donation()); - try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - .values = try value_list.toOwnedSlice(), - .donations = try donation_list.toOwnedSlice(), - .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), - }; -} - -pub const InputInfo = struct { - shapes: []Shape, - - pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { - allocator.free(self.shapes); - } -}; - -fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { - const LocalContext = struct { - shape_list: *std.array_list.Managed(Shape), - }; - - var shape_list = std.array_list.Managed(Shape).init(allocator); - errdefer shape_list.deinit(); - - var context: LocalContext = .{ .shape_list = &shape_list }; - - try meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { - try ctx_.shape_list.append(tensor.shape()); - } - }.cb, &context, v); - - return .{ - .shapes = try shape_list.toOwnedSlice(), - }; -} - -const EmitMlirResult = struct { - func: *mlir.Operation, - input_info: InputInfo, - output_info: OutputInfo, -}; - -fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { - const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); - for (res, attributes) |*r, attr| { - r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); - } - return res; -} - -fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { - var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); - defer arena.deinit(); - - const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); - errdefer module.deinit(); - - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - compilation_context.pushBlock(block); - defer compilation_context.popBlock(); - - const LocalContext = struct { - compilation_context: *CompilationContext, - current_argument_id: usize = 0, - }; - var context: LocalContext = .{ - .compilation_context = compilation_context, - }; - meta.visit(struct { - fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { - const mlir_type = mlir.rankedTensorType( - tensor.dims(), - mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), - ); - _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); - ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; - ctx_.current_argument_id += 1; - } - }.cb, &context, &args); - - const input_info = try collectInputInfo(compilation_context.allocator, &args); - errdefer input_info.deinit(compilation_context.allocator); - - const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); - @memset(input_attributes, .{}); - - const output_info = b: { - compilation_context.activate(); - defer compilation_context.deactivate(); - - const result = @call(.auto, func, args); - - const output_info = try collectOutputInfo(compilation_context.allocator, &result); - errdefer output_info.deinit(compilation_context.allocator); - - break :b output_info; - }; - errdefer output_info.deinit(compilation_context.allocator); - - const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); - @memset(output_attributes, .{}); - - for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { - input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); - }; - for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { - if (output_memory_kind == .device) continue; - output_attributes[index].appendAssumeCapacity(.named( - compilation_context.mlir_ctx, - "mhlo.memory_kind", - mlir.stringAttribute( - compilation_context.mlir_ctx, - compilation_context.platform.memoryKind(output_memory_kind), - ), - )); - } - _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); - - const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ - .name = "main", - .block = compilation_context.currentScope().block, - .location = .unknown(compilation_context.mlir_ctx), - .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), - .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), - }); - - compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { - error.MlirUnexpected => { - std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); - // user errors should have triggered a panic before we reach this. - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - - return .{ - .func = mlir_func, - .input_info = input_info, - .output_info = output_info, - }; -} - -fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { - const result = c.upb_Map_Set( - map, - .{ .str_val = upb.stringView(flag) }, - .{ .msg_val = blk: { - const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); - switch (@typeInfo(@TypeOf(value))) { - .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), - .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), - .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), - else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), - } - break :blk @ptrCast(field); - } }, - upb_arena, - ); - - if (result == false) { - return std.mem.Allocator.Error.OutOfMemory; - } -} - -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { - //const tracer = Tracer.init("ai.zml.compilation"); - //const compile_frame = tracer.frameStart("pjrt compilation"); - //defer tracer.frameEnd(compile_frame, "pjrt compilation"); - - const sharding = platform.sharding(); - - var upb_alloc: upb.Allocator = .init(arena); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const options = blk: { - const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); - c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { - const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); - - c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); - - c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { - const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); - - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); - for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { - computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); - } - break :device_assignment_blk device_assignment; - }); - - break :executable_build_options_blk exec_build_options; - }); - - const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); - switch (platform.target) { - .cuda => { - // NVIDIA recommends these settings - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); - }, - .rocm => { - // enable only on CDNA - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", gemm_blk: { - const supported: std.StaticStringMap(void) = .initComptime(.{ - .{ "gfx950", {} }, - .{ "gfx950", {} }, - .{ "gfx942", {} }, - .{ "gfx942", {} }, - .{ "gfx942", {} }, - .{ "gfx90a", {} }, - .{ "gfx90a", {} }, - .{ "gfx90a", {} }, - .{ "gfx908", {} }, - .{ "gfx906", {} }, - .{ "gfx900", {} }, - }); - break :gemm_blk supported.has(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "compute_capability").?.string); - }, upb_arena); - // Use lld from libllvm instead of invoking the ld.lld binary. - // This saves us from having to sandbox it. - try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); - // Disable command buffer to avoid some weird crashes. - // This is what AMD recommended in the meantime. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); - }, - else => {}, - } - - if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { - try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); - if (platform.compilation_options.xla_dump_fusion_visualization) { - try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); - } - if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); - } - } - - break :blk options; - }; - - const loaded_executable = try pjrtx.Client.compile( - platform.pjrt_client, - platform.pjrt_api, - arena, - io, - module, - try upb.serialize(options, upb_arena), - ); - errdefer loaded_executable.deinit(); - - return loaded_executable; -} diff --git a/zml/nn.zig b/zml/nn.zig index a903c712..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,13 +1,19 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const Slice = @import("slice.zig").Slice; +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); +const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; const zml = @import("zml.zig"); @@ -17,6 +23,7 @@ pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, tag: Shape.Tag, + quantization: ?Quantization = null, pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { return .{ @@ -26,13 +33,331 @@ pub const Linear = struct { }; } - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dot(self.weight, self.tag); + pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { + zml.Buffer.deinitAll(Linear, self); + } - return if (self.bias) |bias| y.add(bias.broad(y.shape())) else y; + pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { + return if (self.quantization) |q| q.scheme else null; + } + + pub fn quantizationScales(self: Linear) ?Tensor { + return if (self.quantization) |q| q.scales else null; + } + + pub fn forward(self: Linear, x: Tensor, out_dtype: DataType) Tensor { + if (self.quantization) |q| { + if (quantization.quantizeInput(q, x, self.tag, zml.Compiler.current().platform)) |input| { + return self.forwardQuantized(input, out_dtype); + } + return self.forwardScaled(x.convert(.bf16), null, null, out_dtype); + } + const y = x.dot(self.weight.convert(x.dtype()), self.tag).convert(out_dtype); + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; + } + + /// Apply this layer to reusable quantized activation values and scales. + /// Convert to output_dtype before adding bias. + pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { + stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); + return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); + } + + fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, output_dtype: DataType) Tensor { + const q = self.quantization.?; + const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; + const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; + const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); + const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } }; +pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); +} + +/// Unpacks two f4e2m1 values per byte along `packed_tag` and names the expanded +/// axis `k_tag`. Group-size agnostic: NVFP4 (16) and MXFP4 (32) share this packing. +pub fn unpackFp4(w: Tensor, packed_tag: anytype, contracting_tag: anytype) Tensor { + stdx.debug.assert(w.dtype() == .u8 or w.dtype() == .i8, "unpackFp4 expects packed 8-bit weights, got {}", .{w.dtype()}); + const source_tag = Shape.toTag(packed_tag); + const result_tag = Shape.toTag(contracting_tag); + return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it + .merge(.{ .kb = .{ source_tag, .bitcast } }) + .renameTag(.kb, result_tag); +} + +test "unpackFp4 expands the requested axis" { + const platform = zml.testing.env(); + const packed_weight: Tensor = .init(.{ .out = 2, .stored = 4 }, .u8); + const Local = struct { + fn call(w: Tensor) Tensor { + return unpackFp4(w, .stored, .contracted); + } + }; + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{packed_weight}, .{}); + defer exe.deinit(); + + try zml.testing.expectEqualShapes(.init(.{ .out = 2, .contracted = 8 }, .f4e2m1), exe.output_shapes[0]); +} + +/// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. +/// +/// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) +/// - **MXFP4**: values `.f4e2m1`, scales `.f8e8m0fnu`, block 32 +/// - **MXFP8**: values `.f8e4m3fn` / `.f8e5m2`, scales `.f8e8m0fnu`, block 32 +/// - **Block FP8**: E4M3FN / E4M3FNUZ / E8M0 values, BF16 or F32 128x128 scales +/// - TODO: INT4/8 and FP8 per tensor +/// +/// Backends: +/// XLA selects specialized GPU kernels (TileIR, Triton, or block-128 W8A8), +/// with dequant + Dot as the fallback. +/// +/// Axes follow `Tensor.dot`; XLA currently accepts one contracting axis and at +/// most one batch axis. Scales preserve operand axis order, and each scale +/// dimension must divide its corresponding value dimension. +/// +/// CPU has no specialized path. +/// +/// `out_dtype` is the element type of the returned tensor; only BF16 and F32 stay on +/// the fused path. It buys no accuracy: the kernels accumulate in F32 either way. +pub fn scaledDot( + lhs: Tensor, + rhs: Tensor, + lhs_scale: ?Tensor, + rhs_scale: Tensor, + out_dtype: DataType, + args: anytype, + opts: ScaledDotOpts, +) Tensor { + // A narrow output would be an unscaled cast of the F32 accumulator + stdx.debug.assert( + out_dtype.isFloat() and out_dtype.bitSizeOf() >= 16, + "scaledDot output dtype must be a 16-bit-or-wider float, got {s}", + .{@tagName(out_dtype)}, + ); + const dot_axes = lhs.dotAxes(rhs, args); + + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); + + var res_shape: Shape = .scalar(out_dtype); + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; + for (dot_axes.batching.constSlice()) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; + for (dot_axes.contracting.constSlice()) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const lhs_scale_operand = lhs_scale orelse blk: { + break :blk Tensor.constantTensor(onesGrid(lhs.shape(), .bf16), DataType.bf16.one().asBytes()); + }; + + const rhs_scale_operand = if (rhs_scale.rank() != rhs.rank() and rhs_scale.shape().count() == 1) + rhs_scale.reshape(onesGrid(rhs.shape(), rhs_scale.dtype())) + else + rhs_scale; + + const mlir_ctx = zml.Compiler.current().mlir_ctx; + const dnums = mlir.Attribute.array(mlir_ctx, &.{ + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_contracting_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_contracting_axes.constSlice()), + }), + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_batching_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), + }), + }); + const attributes = [_]mlir.NamedAttribute{ + .named(mlir_ctx, "dimension_numbers", dnums), + .named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true)), + }; + if (opts.rhs_scale_swizzled) { + stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled rhs scale needs a [n, k] rhs, got {f}", .{rhs}); + } + return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ + .composite_attributes = if (opts.rhs_scale_swizzled) &attributes else attributes[0..1], + })[0]; +} + +pub const ScaledDotOpts = struct { + /// The rhs scale's `[n, k / block]` grid is stored in the 128x4 blocked layout of Blackwell's block-scaled mma, + /// each device's slice on its own. + // TODO: fine for now, but we should find a better way to express how a scale is laid out. + rhs_scale_swizzled: bool = false, +}; + +test "block128 scaled dot layouts" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) + if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; + const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; + + fn dequantize(values: Tensor, scales: Tensor) Tensor { + var expanded = scales.convert(.bf16); + for (0..values.rank()) |axis| { + const factor = @divExact(values.dim(axis), scales.dim(axis)); + const shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(shape, axes.dims()).reshape(expanded.shape().setDim(axis, values.dim(axis))); + } + return values.convert(.bf16).mul(expanded); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, fp8: DataType, prequantized: bool) Outputs { + const weight = w.convert(fp8); + const input = quantization.quantizeBlockFp8(x, .k, 128, fp8, .f32); + const linear: Linear = .{ .weight = weight, .tag = Shape.toTag(.k), .quantization = .{ .scheme = .fp8_block128, .scales = scales } }; + return .{ + .linear = linear.forward(x, .bf16), + .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), + .actual = if (prequantized) + scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k, .{}) + else + scaledDot(x, weight, null, scales, .bf16, .k, .{}), + .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) + .dot(dequantize(weight, scales), .k), + }; + } + }; + inline for (.{ + .{ .{ .m = 16, .k = 256 }, .{ .n = 256, .k = 256 } }, + .{ .{ .b = 2, .m = 16, .k = 256 }, .{ .b = 2, .n = 256, .k = 256 } }, + .{ .{ .k = 256, .b = 2, .m = 3 }, .{ .n = 256, .k = 256, .b = 2 } }, + .{ .{ .b = 2, .k = 256, .s = 2, .m = 3 }, .{ .n = 128, .b = 2, .k = 256 } }, + .{ .{ .b = 2, .m = 3, .k = 256 }, .{ .k = 256, .n = 128 } }, + .{ .{ .k = 128 }, .{ .n = 128, .k = 128 } }, + .{ .{ .m = 3, .k = 256 }, .{ .p = 2, .k = 256, .n = 128 } }, + .{ .{ .m = 3, .k = 128 }, .{ .n = 64, .k = 128 } }, + }) |layout| { + inline for (.{ DataType.f32, DataType.bf16 }) |scale_dtype| { + inline for (.{ false, true }) |prequantized| { + const x: Tensor = .init(layout[0], .bf16); + const w: Tensor = .init(layout[1], .bf16); + const scales: Tensor = .init(w.shape().setDim(.n, std.math.divCeil(i64, w.dim(.n), 128) catch unreachable).setDim(.k, @divExact(w.dim(.k), 128)).withDtype(scale_dtype), scale_dtype); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, dtype, prequantized }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(exe.output_shapes[1], exe.output_shapes[0]); + var buffers: [3]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape() }, 0..) |shape, i| { + const slice = try Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + const value: f32 = if (i == 2) @as(f32, @floatFromInt(1 + (j * 3 + j / 4) % 7)) / 16 else @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var expected = try output.expected.toSliceAlloc(allocator, io); + defer expected.free(allocator); + try zml.testing.expectClose(io, expected, output.actual, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + var linear_expected = try output.linear_expected.toSliceAlloc(allocator, io); + defer linear_expected.free(allocator); + try zml.testing.expectClose(io, linear_expected, output.linear, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + } + } + } +} + +/// `shape` with every dimension collapsed to 1 +fn onesGrid(shape: Shape, dt: DataType) Shape { + var res = shape.withDtype(dt); + for (0..res.rank()) |i| { + res = res.setDim(i, 1); + } + + return res; +} + +fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); +} + +fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { + if (igs == null and wgs == null) { + return acc; + } + + var res = acc.convert(.f32); + if (wgs) |w| { + res = res.mul(w.convert(.f32).broad(res.shape())); + } + + if (igs) |i| { + res = res.mul(i.convert(.f32).broad(res.shape())); + } + + return res.convert(acc.dtype()); +} + pub const TokenEmbedding = struct { weight: Tensor, @@ -127,29 +452,32 @@ test normalizeL2 { const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); defer res.deinit(); const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } pub const RopeOpts = struct { - layout: Layout = .sequential, - freq_base: f32 = 10_000, + layout: Layout = .real_im_pass, scaling: Scaling = .{ .default = .{} }, - /// There are two layouts corresponding to how to split `x` in real/imag parts. - /// * Interleaved means that the real/imag of each scalar is contiguous. - /// * Sequential means that you first read all real values then all imag values. - /// Typically HF models use sequential, while GGUF use interleaved. - pub const Layout = enum { interleaved, sequential }; + /// There are 3 layouts corresponding to how to split `x` in real/imag/passthrough parts. + /// The hard part is that HF models don't specify the layout they use. + /// But HF models don't use the `interleaved` format, + /// and most models don't have a Rope passthrough, so real_im_pass and real_pass_im_pass are the same. + /// + /// * real_im_pass: [x_re0, x_re1, ..., x_im0, x_im1, ..., p_re0, p_re1, ..., p_im0, p_im1, ...] + /// * real_pass_im_pass: [x_re0, x_re1, ..., p_re0, p_re1, ..., x_im0, x_im1, ..., p_im0, p_im1, ...] + /// * interleaved: : [x_re0, x_im0, x_re1, x_im1, ..., p_re0, p_im0, p_re1, p_im1, ...] + pub const Layout = enum { real_im_pass, real_pass_im_pass, interleaved }; /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { @@ -157,6 +485,7 @@ pub const RopeOpts = struct { llama3: Llama3, yarn: Yarn, linear: Scaling.Linear, + proportional: Scaling.Proportional, pub const Default = struct { rope_theta: f32 = 10000, @@ -179,9 +508,10 @@ pub const RopeOpts = struct { mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, - rope_theta: f32, + rope_theta: f32 = 10000, llama_4_scaling_beta: ?f32 = null, attention_factor: ?f32 = null, + partial_rotary_factor: f32 = 1.0, }; pub const Linear = struct { @@ -189,6 +519,11 @@ pub const RopeOpts = struct { rope_theta: f32 = 10000, }; + pub const Proportional = struct { + partial_rotary_factor: f32 = 1.0, + rope_theta: f32 = 10000, + }; + /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -199,8 +534,8 @@ pub const RopeOpts = struct { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { - _ = allocator; // autofix - _ = options; // autofix + _ = allocator; + _ = options; if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; @@ -211,10 +546,14 @@ pub const RopeOpts = struct { if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "default")) { + return .{ .default = try std.json.parseFromValueLeaky(Default, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "linear")) { return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "proportional")) { + return .{ .proportional = try std.json.parseFromValueLeaky(Scaling.Proportional, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = .{} }; @@ -256,6 +595,15 @@ pub const RopeOpts = struct { inline else => |s| s.rope_theta, }; } + + pub fn partialRotaryDim(self: *const Scaling, head_dim: i64) u32 { + return switch (self.*) { + inline .yarn, .proportional => |s| @intFromFloat(@floor( + @as(f32, @floatFromInt(head_dim)) * std.math.clamp(s.partial_rotary_factor, 0.0, 1.0), + )), + .default, .llama3, .linear => @intCast(head_dim), + }; + } }; }; @@ -269,7 +617,8 @@ pub const RopeOpts = struct { /// - pos_idx: optional tensor which indicates which positions are needed. /// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { - stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + const head_dim = x.dim(.hd); + stdx.debug.assert(@mod(head_dim, 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); const idx = if (pos_idx) |idx| blk: { stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); @@ -278,11 +627,15 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; - const x_real, const x_imag = splitRealImg(x, opts.layout); + + const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); + stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); + + const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); + const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); - const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); + const inv_freq_pos = zml.Tensor.outer(idx.convert(.f32), inv_freq); const scaling = opts.scaling.attentionScaling(); const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); @@ -291,55 +644,116 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - // flatten last dimensions - return mergeRealImg(y_real, y_imag, opts.layout); + return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); } pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); return switch (layout) { - .sequential => .{ - x.slice1d(-1, .{ .end = @divExact(n, 2) }), - x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + .real_im_pass, .real_pass_im_pass => .{ + x.slice(-1, .{ .end = @divExact(n, 2) }), + x.slice(-1, .{ .start = @divExact(n, 2), .end = n }), }, .interleaved => .{ - x.slice1d(-1, .{ .start = 0, .step = 2 }), - x.slice1d(-1, .{ .start = 1, .step = 2 }), + x.slice(-1, .{ .start = 0, .step = 2 }), + x.slice(-1, .{ .start = 1, .step = 2 }), + }, + }; +} + +const Pass = union(RopeOpts.Layout) { + real_im_pass: Tensor, + real_pass_im_pass: [2]Tensor, + interleaved: Tensor, +}; + +pub fn splitRealImgPass(x: Tensor, layout: RopeOpts.Layout, rotary_dim: u32) struct { Tensor, Tensor, ?Pass } { + const ax = x.axis(.hd); + const hd = x.dim(ax); + if (rotary_dim == hd) { + const real, const img = splitRealImg(x, layout); + return .{ real, img, null }; + } + + const half = @divExact(x.dim(ax), 2); + const half_rotary = @divExact(rotary_dim, 2); + return switch (layout) { + .real_im_pass => .{ + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half_rotary, .end = rotary_dim }), + .{ .real_im_pass = x.slice(ax, .{ .start = rotary_dim }) }, + }, + .real_pass_im_pass => .{ + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half, .end = half + half_rotary }), + .{ .real_pass_im_pass = .{ + x.slice(ax, .{ .start = half_rotary, .end = half }), + x.slice(ax, .{ .start = half + half_rotary }), + } }, + }, + .interleaved => .{ + x.slice(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), + x.slice(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), + .{ .interleaved = x.slice(ax, .{ .start = rotary_dim }) }, }, }; } pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { return switch (layout) { - .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), - .interleaved => Tensor.concatenate(&.{ - x_real.appendAxes(.{.interleaved_real_img}), - x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).reshape(x_imag.shape().setDim(-1, -1)), + .real_im_pass, .real_pass_im_pass => .concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + }; +} + +pub fn mergeRealImgPass(x_real: Tensor, x_imag: Tensor, x_pass: ?Pass, layout: RopeOpts.Layout) Tensor { + if (x_pass == null) return mergeRealImg(x_real, x_imag, layout); + + return switch (x_pass.?) { + .real_im_pass => |pass| .concatenate(&.{ x_real, x_imag, pass }, .hd), + .real_pass_im_pass => |pass| .concatenate(&.{ x_real, pass[0], x_imag, pass[1] }, .hd), + .interleaved => |pass| .concatenate( + &.{ + Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + pass, + }, + .hd, + ), }; } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -fn invFreq(N: i64, opts: RopeOpts) Tensor { - const allocator = zml.module.CompilationContext.current().allocator; - const N_half: usize = @intCast(@divExact(N, 2)); - const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); +pub fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.Compiler.current().allocator; + const N_half: u32 = @intCast(@divExact(N, 2)); + const num_freqs: u32 = opts.scaling.partialRotaryDim(N_half); + + const inv_freq = allocator.alloc(f32, num_freqs) catch @panic("OOM"); defer allocator.free(inv_freq); - _invFreq(opts, inv_freq); - return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); + _invFreq(opts, inv_freq, N_half); + return .constantTensor(.init(.{num_freqs}, .f32), @ptrCast(inv_freq)); } -fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { - const N = inv_freq.len; +fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { + const N: u32 = @intCast(inv_freq.len); // Default frequencies for (0.., inv_freq) |n, *f| { - f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, N)); } switch (opts.scaling) { .default => {}, + .proportional => { + // proportional uses the original number of dimensions, + // instead of the actual number of rotated dims. + for (0.., inv_freq[0..]) |n, *f| { + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, half_head_dim)); + } + }, .linear => |l| { for (inv_freq) |*f| f.* /= l.factor; }, @@ -367,15 +781,15 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { } }, .yarn => |s| { - const N_f: f64 = @floatFromInt(inv_freq.len); + const N_f: f64 = @floatFromInt(N); const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. - var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; - var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; + var n_low: f64 = -@log(f_high) / @log(opts.scaling.getRopeTheta()) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.scaling.getRopeTheta()) * N_f; if (s.truncate) { n_high = std.math.ceil(n_high); n_low = std.math.floor(n_low); @@ -400,18 +814,18 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ - .freq_base = 500_000, .scaling = .{ .llama3 = .{ .factor = 32, .high_freq_factor = 4, .low_freq_factor = 1, .original_max_position_embeddings = 8192, + .rope_theta = 500_000, } }, }; const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; var inv_freq: @TypeOf(llama_freq) = undefined; - _invFreq(llama_conf, &inv_freq); + _invFreq(llama_conf, &inv_freq, 32); for (llama_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -420,19 +834,19 @@ test "invFreq Llama3" { test "invFreq Yarn" { const yarn_conf: RopeOpts = .{ - .freq_base = 150_000, .scaling = .{ .yarn = .{ .factor = 32.0, .beta_fast = 32.0, .beta_slow = 1.0, .original_max_position_embeddings = 4096, .truncate = true, + .rope_theta = 150_000, } }, }; const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; var inv_freq: @TypeOf(yarn_freq) = undefined; - _invFreq(yarn_conf, &inv_freq); + _invFreq(yarn_conf, &inv_freq, 32); for (yarn_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -440,12 +854,73 @@ test "invFreq Yarn" { try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); } +test "RopeOpts.Scaling parses proportional" { + const json = + \\{ + \\ "rope_type": "proportional", + \\ "partial_rotary_factor": 0.25, + \\ "rope_theta": 1000000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .proportional => |p| { + try std.testing.expectApproxEqRel(0.25, p.partial_rotary_factor, 1e-6); + try std.testing.expectApproxEqRel(1_000_000, p.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "RopeOpts.Scaling parses yarn partial rotary factor" { + const json = + \\{ + \\ "rope_type": "yarn", + \\ "factor": 32.0, + \\ "original_max_position_embeddings": 4096, + \\ "partial_rotary_factor": 0.5 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .yarn => |y| { + try std.testing.expectApproxEqRel(0.5, y.partial_rotary_factor, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "RopeOpts.Scaling parses default rope type" { + const json = + \\{ + \\ "rope_type": "default", + \\ "rope_theta": 10000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .default => |d| { + try std.testing.expectApproxEqRel(10_000, d.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } +} + test "real/img" { const platform = zml.testing.env(); const Fns = struct { fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); @@ -455,8 +930,8 @@ test "real/img" { } fn testSplitSeqVoid(_: void) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -473,7 +948,7 @@ test "real/img" { fn testSplitSeq() Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -500,47 +975,47 @@ test "real/img" { } }; { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, .{}); defer exe.deinit(); - var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + var d_interleaved = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_interleaved.deinit(); try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, .{}); defer exe.deinit(); - var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); + var d_sequential = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_sequential.deinit(); try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); } // test the function that accepts 1 void argument { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, .{}); defer exe.deinit(); - var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); + var d_split_seq_void = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq_void.deinit(); try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); } // test the function that takes NO arguments { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, .{}); defer exe.deinit(); - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, .{}); defer exe.deinit(); - var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } @@ -554,7 +1029,7 @@ test rope { var input = x; { // Convert input to the requested format - const real, const imag = splitRealImg(input, .sequential); + const real, const imag = splitRealImg(input, .real_im_pass); input = mergeRealImg(real, imag, opts.layout); } var res = rope(input, null, opts).squeeze(0); @@ -562,7 +1037,7 @@ test rope { { // Convert back to sequential const real, const imag = splitRealImg(res, opts.layout); - res = mergeRealImg(real, imag, .sequential); + res = mergeRealImg(real, imag, .real_im_pass); } return res; } @@ -571,21 +1046,118 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); + var exe_interleaved = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, .{}); defer exe_interleaved.deinit(); - var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); + var exe_sequential = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, .{}); defer exe_sequential.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); + const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); defer x_buffer.deinit(); - var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); + var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res1.deinit(); - var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); + var res2 = try exe_sequential.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res2.deinit(); - try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); + try zml.testing.expectClose(std.testing.io, res1, res2, .{}); +} + +test "rope: Proportional" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try platform.compileFn( + allocator, + io, + rope, + .{ + x, + null, + .{ + .layout = .real_pass_im_pass, + .scaling = .{ .proportional = .{ .partial_rotary_factor = 0.25 } }, + }, + }, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.667, -0.449, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.468, 0.904, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.0328, -0.708, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.814, 0.719, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.703, -0.897, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.412, 0.464, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.727, -0.997, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.369, 0.162, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try exe.eval(allocator, io, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); +} + +test "rope: Yarn with partial_rotary_factor" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try platform.compileFn( + allocator, + io, + rope, + .{ + x, + null, + RopeOpts{ + .layout = .real_im_pass, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + .rope_theta = 150_000, + .partial_rotary_factor = 0.25, + } }, + }, + }, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 1.02, -0.197, 0.00597, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.544, -0.195, 0.858, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.428, -0.193, 0.921, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -1.01, -0.191, 0.137, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.659, -0.19, -0.772, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try exe.eval(allocator, io, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); } pub const UpsampleMode = enum { @@ -650,24 +1222,41 @@ test nearest { // 3D Tensor (basic) { const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, + .{}, + ); defer exe.deinit(); - var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); + var input_3d_basic_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input_3d_basic.shape(), + std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), + ); defer input_3d_basic_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); const expected: Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 3D Tensor (advanced) { const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, + .{}, + ); defer exe.deinit(); var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ @@ -676,7 +1265,7 @@ test nearest { })); defer input_3d_advanced_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); @@ -692,19 +1281,25 @@ test nearest { .{ 21, 21, 22, 22, 23, 23, 24, 24 }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 4D Tensor (basic) { const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, + .{}, + ); defer exe.deinit(); var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); @@ -716,12 +1311,18 @@ test nearest { .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, }}})); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 4D Tensor (advanced) { const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, + .{}, + ); defer exe.deinit(); var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ @@ -733,7 +1334,7 @@ test nearest { } })); defer input_4d_advanced_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); @@ -767,18 +1368,24 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 5D Tensor (basic) { const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, + .{}, + ); defer exe.deinit(); var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); @@ -800,7 +1407,7 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } } @@ -823,7 +1430,7 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeLinear1d(out, ax, d, child_opt); } @@ -832,15 +1439,16 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten test resizeBilinear { const platform = zml.testing.env(); + // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, @@ -889,7 +1497,7 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeCubic1d(out, ax, d, child_opt); } @@ -898,15 +1506,16 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens test resizeBicubic { const platform = zml.testing.env(); + // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, @@ -998,9 +1607,7 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); - const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); - mask = Tensor.select(mask, zeros, minus_inf); + mask = Tensor.select(mask, .scalar(0, dtype), .scalar(-std.math.inf(f32), dtype)); } else { mask = mask.convert(dtype); } @@ -1012,9 +1619,6 @@ pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, softmax_bias: ?Tensor = null, - allow_cudnn: bool = true, - // TODO: put a callback instead of all this field, - // so that }; /// Scaled dot product attention. @@ -1040,11 +1644,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - // TODO(Corentin): Re-enable that - //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { - // return cuda.sdpa(q, k, v, opts); - //} - // Handle different numbers of head by splitting q heads. // This is a bit error prone in the sense that it depends of the layout of q heads. // This is the Llama convention though. @@ -1059,7 +1658,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .hd); - // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { @@ -1073,6 +1671,308 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } +pub const GatedDeltaNet = struct { + /// Query tensor .{ .s, .h, .k }. + queries: Tensor, + /// Key tensor .{ .s, .h, .k }. + keys: Tensor, + /// Value tensor .{ .s, .h, .v }. + values: Tensor, + /// Forget gate .{ .s, .h }. + alphas: Tensor, + /// Delta gate .{ .s, .h }. + betas: Tensor, + /// Number of sequence positions to process, clamped to the input length. + active_length: Tensor, + + pub const State = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + s: Tensor, + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, + + /// Current step, scalar + step: Tensor, + }; + + pub const Input = struct { + s: Tensor, + }; + + pub const Output = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + state: Input, + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, + }; + + pub fn cond(gdn: GatedDeltaNet, state: State) Tensor { + const active_length = gdn.active_length + .convert(.i32) + .maximum(.scalar(0, .i32)) + .minimum(.scalar(gdn.queries.dim(.s), .i32)); + return state.step.cmp(.LT, active_length); + } + + /// Single-step recurrent update for Gated Delta Net. + pub fn body(gdn: GatedDeltaNet, state: State) State { + // q, k: .{ .h, .k } + const q = sliceStep(gdn.queries, state.step); + const k = sliceStep(gdn.keys, state.step); + // v: .{ .h, .v } + const v = sliceStep(gdn.values, state.step); + // alpha, beta: .{ .h } + const alpha = sliceStep(gdn.alphas, state.step); + const beta = sliceStep(gdn.betas, state.step); + + const hvk = state.s.shape(); + const hv = hvk.drop(.k); + + const v_hat = state.s.dot(k, .k).mul(alpha.broad(hv)); + const delta = v.sub(v_hat).mul(beta.broad(hv)); + const delta_k = delta.broad(hvk).mul(k.broad(hvk)); + const s_new = state.s.mul(alpha.broad(hvk)).add(delta_k); + const y = s_new.dot(q, .k); + + return .{ + .s = s_new, + .outputs = state.outputs.dynamicUpdateSlice(.{ .s = state.step }, y), + .step = state.step.addConstant(1), + }; + } + + fn sliceStep(input: Tensor, step_: Tensor) Tensor { + return input.slice(.s, .dynSingle(step_)); + } + + fn validateInitialState(gdn: GatedDeltaNet, state: State) void { + const err_template = "GatedDeltaNet.step(state: {f}, q: {f}, k: {f}, v: {f}, alpha: {f}, beta: {f}) is invalid ! "; + const err_args = .{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }; + + stdx.debug.assert(state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "state.s is missing tags {{.h, .v, .k}}", err_args); + stdx.debug.assert(gdn.queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); + stdx.debug.assert(gdn.alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.active_length.rank() == 0 and gdn.active_length.dtype().isInteger(), err_template ++ "active_length must be an integer scalar", err_args); + + _ = collectDims(.{.h}, &.{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }, .strict) catch { + stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.k}, &.{ state.s, gdn.queries, gdn.keys }, .strict) catch { + stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.v}, &.{ state.s, gdn.values }, .strict) catch { + stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); + }; + } + + /// Processes a sequence with Gated Delta Net recurrence using `stablehlo.while`. + /// + /// Shapes: + /// - `queries`, `keys`: .{ .s, .h, .k } + /// - `values`, `outputs`: .{ .s, .h, .v } + /// - `alphas`, `betas`: .{ .s, .h } + /// - `initial_state.s`: .{ .h, .v, .k } + /// - `active_length`: integer scalar + pub fn forward( + queries: Tensor, + keys: Tensor, + values: Tensor, + alphas: Tensor, + betas: Tensor, + initial_state: Input, + active_length: Tensor, + ) Output { + const gdn: GatedDeltaNet = .{ + .queries = queries, + .keys = keys, + .values = values, + .alphas = alphas, + .betas = betas, + .active_length = active_length, + }; + const state: GatedDeltaNet.State = .{ + .step = .scalar(0, .i32), + .s = initial_state.s, + .outputs = .zeroes(values.shape()), + }; + validateInitialState(gdn, state); + + const final_state = if (queries.dim(.s) == 1) blk: { + // avoids a `stablehlo.while`, which costs a host sync per call on GPU. + const next = gdn.body(state); + const run = gdn.cond(state); + break :blk GatedDeltaNet.State{ + .s = run.broad(state.s.shape()).select(next.s, state.s), + .outputs = run.broad(state.outputs.shape()).select(next.outputs, state.outputs), + .step = run.select(next.step, state.step), + }; + } else ops.@"while"(GatedDeltaNet, gdn, state); + + return .{ + .outputs = final_state.outputs, + .state = .{ .s = final_state.s }, + }; + } +}; + +test "gated delta net" { + const platform = zml.testing.env(); + + const queries: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const keys: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const values: zml.Tensor = .init(.{ .s = 2, .h = 2, .v = 2 }, .f32); + const alphas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); + const active_length: zml.Tensor = .init(.{}, .u32); + + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + GatedDeltaNet.forward, + .{ queries, keys, values, alphas, betas, .{ .s = initial_s }, active_length }, + .{}, + ); + defer exe.deinit(); + + var queries_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + queries.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, + }), + ); + defer queries_buffer.deinit(); + var keys_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + keys.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer keys_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + values.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, + .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, + }), + ); + defer values_buffer.deinit(); + var alphas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + alphas.shape(), + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 0.5, 0.25 }, + .{ 0.8, 0.6 }, + }), + ); + defer alphas_buffer.deinit(); + var betas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + betas.shape(), + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 1.0, 0.5 }, + .{ 0.75, 1.0 }, + }), + ); + defer betas_buffer.deinit(); + var initial_s_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + initial_s.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer initial_s_buffer.deinit(); + var oversized_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 3), .u32); + defer oversized_length_buffer.deinit(); + + var result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run( + std.testing.io, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, oversized_length_buffer }, + .{&result}, + .{ .wait = true }, + ); + defer result.outputs.deinit(); + defer result.state.s.deinit(); + + const expected_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ -11.15, 10.75 }, .{ 2.325, -1.2 } }, + }), + ); + const expected_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ -9.3, -1.85 }, .{ 6.9, 3.85 } }, + .{ .{ 3.0, 0.675 }, .{ 0.0, 1.2 } }, + }), + ); + + try zml.testing.expectClose(std.testing.io, expected_outputs, result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_final_s, result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + + var prefix_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 1), .u32); + defer prefix_length_buffer.deinit(); + var prefix_result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + runner.run( + std.testing.io, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, prefix_length_buffer }, + .{&prefix_result}, + .{ .wait = true }, + ); + defer prefix_result.outputs.deinit(); + defer prefix_result.state.s.deinit(); + + const expected_prefix_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ 0.0, 0.0 }, .{ 0.0, 0.0 } }, + }), + ); + const expected_prefix_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 5.0 }, .{ 0.0, 0.5 } }, + .{ .{ 0.5, 1.125 }, .{ 0.25, 2.0 } }, + }), + ); + try zml.testing.expectClose(std.testing.io, expected_prefix_outputs, prefix_result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_prefix_final_s, prefix_result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); +} + /// Options controlling generation. The default values correspond to greedy decoding. pub const SamplingStrategy = struct { topk: u32 = 1, @@ -1116,10 +2016,10 @@ test sampleTokens { const rng: zml.Tensor.Rng = .init(); const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); defer exe.deinit(); - var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); + var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -1134,7 +2034,10 @@ test sampleTokens { var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); - var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); + var sampled: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ activations_buffer, rng_buffer }, .{ &sampled, &rng_buffer }, .{ .wait = true }); defer sampled.deinit(); try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); @@ -1223,17 +2126,19 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // After the topk, we don't have .voc indices, anymore, only topk. var x = full_topk.values.rename(.{ .voc = .topk }); // mask values above the dynamic top_k - x = Tensor.iota(x.shape(), .topk).cmp(.GE, opts.top_k).select(min_inf, x); - x = x.mul(opts.temperature); + const top_k = opts.top_k.broad(x.shape()); + x = Tensor.iota(x.shape(), .topk).cmp(.GE, top_k).select(min_inf, x); + const temperature = opts.temperature.convert(x.dtype()).broad(x.shape()); + x = x.div(temperature); // if there are high values in x, softmax can overflow and will create nans in full probs // this propagate to probs_sum and probs_max. const probs = x.softmax(.topk); const probs_sum = probs.cumulativeSum(.topk); - const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); + const probs_max = probs.slice(.topk, .{ .start = 0, .end = 1 }); - const top_p = opts.top_p.broad(x.shape()); - const min_p = probs_max.mul(opts.min_p).broad(x.shape()); + const top_p = opts.top_p.convert(x.dtype()).broad(x.shape()); + const min_p = opts.min_p.convert(x.dtype()).broad(probs_max.shape()).mul(probs_max).broad(x.shape()); // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false // * if first candidate score is even bigger, the probs become Nan because of the softmax, @@ -1256,7 +2161,7 @@ test sampleTokensDynamic { const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); defer exe.deinit(); var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); @@ -1267,7 +2172,7 @@ test sampleTokensDynamic { // top_k == logits.len -> just sort the input .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) * 0.1, @log(3.0) * 0.1, ___, ___ } }, + .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) / 0.1, @log(3.0) / 0.1, ___, ___ } }, // top_k == logits.len and small top_p -> make sure at least one is returned .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, @@ -1280,7 +2185,11 @@ test sampleTokensDynamic { const args, const expected = args_expected; var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_buffer, dynamic_sampling_strategy_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); defer new_logits.deinit(); defer indices.deinit(); try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); @@ -1293,7 +2202,7 @@ test sampleTokensDynamic { const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); + var exe_bf16 = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, .{}); defer exe_bf16.deinit(); const boost = bf16.inf; @@ -1304,7 +2213,13 @@ test sampleTokensDynamic { var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); - var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe_bf16.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); + defer new_logits.deinit(); + defer indices.deinit(); try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); @@ -1366,8 +2281,13 @@ pub fn collectDims( return context.res; } -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); +fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { + const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); + var result: [field_names.len]i64 = undefined; + inline for (field_names, 0..) |field_name, i| { + result[i] = @field(shape, field_name); + } + return result; } test collectDims { @@ -1422,7 +2342,7 @@ fn ShapeStruct(comptime dims: anytype) type { @setEvalBranchQuota(rank + 5); var struct_field_names: [rank][]const u8 = undefined; var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; + var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; const default: i64 = NOT_SET; for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { field_name.* = @tagName(axis); @@ -1435,3 +2355,53 @@ fn ShapeStruct(comptime dims: anytype) type { return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); } + +/// Smooth approximation of ReLU function: log(1 + exp(x)) +/// For numerical stability, falls back to the linear function when `x > opts.threshold`. +pub fn softplus( + x: Tensor, + opts: struct { threshold: f32 = 20.0 }, +) zml.Tensor { + // Compute log1p(z) as z * log(u) / (u - 1), where u = 1 + z. + // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. + const z = x.exp(); + const u = z.addConstant(1); + const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); +} + +test "softplus" { + const inputs = [_]f32{ -20.0, -1.0, 0.0, 1.0, 2.0, 2.1, 3.0 }; + const expected = [_]f32{ 2.0611537e-9, 0.3132617, 0.6931472, 1.3132616, 2.126928, 2.1, 3.0 }; + + const platform = zml.testing.env(); + const input: Tensor = .init(.{ .x = inputs.len }, .f32); + const Local = struct { + fn call(x: Tensor, dtype: DataType) Tensor { + return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); + } + }; + + var input_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input.shape(), + std.mem.sliceAsBytes(&inputs), + ); + defer input_buffer.deinit(); + + for ([_]DataType{ .f32, .bf16 }) |dtype| { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); + defer exe.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); + } +} diff --git a/zml/ops.zig b/zml/ops.zig index 81b5022a..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,29 +1,326 @@ const std = @import("std"); +const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); +const func = @import("mlir/dialects/func"); const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const CompilationContext = @import("module.zig").CompilationContext; +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("mem.zig").Bufferized; +const Compiler = @import("Compiler.zig"); const constants = @import("constants.zig"); +const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; +const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; +const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; +const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; +const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; + +test { + _ = manual_computation; +} + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Reduces local tensor values across all processes in the computation. +/// For each tensor index, `func_` combines the values at that index from every +/// process. The result is identical on all processes. +/// +/// Inside a manual computation, `inputs` describes the local tensors at each +/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives +/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. +/// +/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. +/// Each input must have matching shapes and dtypes across processes. +/// * `func_` describes how to combine two scalar contributions: +/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such +/// as `Tensor.add`. This also applies to a one-element tuple or array. +/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a +/// tuple or array with one scalar tensor per input, in the same order. +/// +/// Returns the same container type as `inputs`, preserving tensor attributes. +/// With one process, returns `inputs` unchanged. +/// +/// The reduction uses one group containing all processes, including mesh replicas. +/// To reduce along one logical mesh axis, use `allReduceOnAxis`. +pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const count = Compiler.current().meshes[0].numDevices(); + var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); + return allReduceGroups(inputs, func_, devices[0..count], count); +} + +/// Reduces local tensor values within process groups defined by `axis` in `mesh`. +/// A group contains processes with the same coordinates on every other logical +/// mesh axis. Separate replicas of the mesh belong to separate groups. Within +/// each group, `func_` combines corresponding elements and the result is identical +/// on all member processes. +/// +/// Use this inside a manual computation to combine partial results along `axis` +/// while preserving the independent computations along the other mesh axes. +/// +/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. +/// * `mesh` is the logical mesh used by the manual computation and must be +/// included in the compilation's meshes. +/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum +/// literal or a mesh axis tag. The axis must exist in `mesh`. +/// +/// Returns the same container type with tensor attributes as +/// `inputs`. An axis of size one returns `inputs` unchanged. +/// +/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local +/// expert contributions independently for each of the two data shards: +/// +/// ```zig +/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); +/// ``` +/// +/// Each expert shard now has the complete output for its data shard. To expose +/// this result through `manualComputation`, an output spec such as +/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the +/// result replicated along `.experts`. +pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); + const size = mesh.grid.axis_sizes[logical_axis]; + var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + const replicas = mesh.numReplicas(); + for (0..mesh.numDevices()) |device| { + const coords = mesh.grid.logical_coords[device]; + var tile: usize = 0; + var group: usize = 0; + for (0..mesh.logical.axes.len) |dim| { + tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; + if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; + } + group = group * replicas + replica_indices[tile]; + replica_indices[tile] += 1; + groups[group * size + coords[logical_axis]] = @intCast(device); + } + return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); +} + +/// Reduces corresponding tensor elements independently within each process group. +/// `inputs`, `func_` and the return value follow `allReduce`'s contract. +/// +/// `groups` identifies processes by their executable partition IDs, as returned +/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. +/// +/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions +/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group +/// receive that group's result. Groups of size one return `inputs` unchanged. +fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { + const ctx = Compiler.current(); + const mlir_ctx = ctx.mlir_ctx; + + const InputsT = @TypeOf(inputs); + const n_inputs = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) break :b 1; + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); + } + break :b struct_info.field_names.len; + }, + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b array_info.len; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + comptime stdx.debug.assertComptime(n_inputs > 0, "zml.ops.allReduce requires at least one input tensor", .{}); + + const input_tensors: [n_inputs]Tensor = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) { + break :b .{inputs}; + } + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); + } + + var flat_inputs: [n_inputs]Tensor = undefined; + meta.collectBuf((struct { + pub fn cb(t: Tensor) Tensor { + return t; + } + }).cb, {}, &inputs, &flat_inputs); + break :b flat_inputs; + }, + .array => inputs, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + + if (group_size <= 1) return inputs; + + const reducer_block = b: { + const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); + var args: @Tuple(&ArgsTypes) = undefined; + var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; + + inline for (0..input_tensors.len) |i| { + const scalar_shape = Shape.init(.{}, input_tensors[i].dtype()); + args[i].left = .fromShape(scalar_shape); + args[i].right = .fromShape(scalar_shape); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); + block_types[i + input_tensors.len] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); + } + + const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(ctx.unknown_location); + + const block = mlir.Block.init(&block_types, &block_locs); + errdefer block.deinit(); + + const reducer_scope = ctx.pushBlock(block); + defer reducer_scope.pop(); + + ctx.pushLocation(@src(), stdx.meta.fnName(func_)); + defer ctx.popLocation(); + + inline for (0..input_tensors.len) |i| { + reducer_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reducer_scope.registerTensorAsBlockArgument(args[i].right.id, i + input_tensors.len); + } + + var reduced_values: [input_tensors.len]*const mlir.Value = undefined; + if (input_tensors.len == 1) { + const reduced = @call(.auto, func_, .{ args[0].left, args[0].right }); + reduced_values[0] = reduced.value(); + } else { + var reduced = @call(.auto, func_, args); + var reduced_tensors: [input_tensors.len]Tensor = undefined; + meta.collectBuf(stdx.meta.identity, {}, &reduced, &reduced_tensors); + + for (reduced_values, reduced_tensors) |*vi, ti| { + vi.* = ti.value(); + } + } + + _ = stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); + break :b block; + }; + + const replica_groups_attr = mlir.Attribute.denseElements( + .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), + groups, + ); + + const handle = ctx.nextChannelId(); + const channel_handle_attr = stablehlo.channelHandleAttr( + mlir_ctx, + handle, + .collective, + ); + + var input_values: [input_tensors.len]*const mlir.Value = undefined; + var result_types: [input_tensors.len]*const mlir.Type = undefined; + inline for (0..input_tensors.len) |i| { + input_values[i] = input_tensors[i].value(); + result_types[i] = input_values[i].type_(); + } + + const op = stablehlo.allReduce( + mlir_ctx, + &input_values, + &result_types, + reducer_block, + replica_groups_attr, + channel_handle_attr, + ctx.location, + ).appendTo(ctx.currentScope().block); + + return switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b Tensor._result(input_tensors[0].shape(), op.result(0)); + } + + var out: @TypeOf(inputs) = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + .array => |array_info| b: { + var out: [array_info.len]Tensor = undefined; + inline for (0..array_info.len) |i| { + out[i] = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + else => unreachable, + }; +} + +fn AllReduceReturnType(comptime InputsT: type) type { + if (InputsT == Tensor) return Tensor; + + return switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor or [N]Tensor inputs"); + } + break :b InputsT; + }, + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b InputsT; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; +} + +/// Returns the current process's zero based coordinate on `axis` in `mesh` +/// as a scalar u32. +/// +/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates +/// 0 and 1. Use this coordinate inside a manual computation to identify a shard +/// along one axis while `partitionId()` identifies a partition across the whole mesh. +pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); + var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; + for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; + return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) + .gather(.{ .device = partitionId() }, .{}); +} + +pub fn partitionId() Tensor { + const ctx = Compiler.current(); + const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); + return Tensor._result(.init(.{}, .u32), op.result(0)); +} pub const ReduceArgs = struct { left: Tensor, right: Tensor, }; -pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func_: anytype, context: anytype) stdx.meta.FnReturn(func_) { + const compiler = Compiler.current(); + const caller_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + const ArgsTypes: [inits.len]type = @splat(ReduceArgs); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -31,58 +328,50 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inits.len]*const mlir.Location = @splat(compiler.unknown_location); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); + + compiler.pushLocation(@src(), stdx.meta.fnName(func_)); + defer compiler.popLocation(); - const scope = CompilationContext.current().currentScope(); inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); var result_values: [inits.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| { result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + const block_result = stablehlo.returns(mlir_ctx, &result_values, compiler.location); + _ = block_result.appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - input_values[i] = inputs[i].value(); - } + inline for (0..inputs.len) |i| input_values[i] = inputs[i].value(); var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + const reduce_op = stablehlo.reduce(mlir_ctx, &input_values, &init_values, reduce_block, axes_, compiler.location).appendTo(caller_scope.block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (0..constants.MAX_RANK) |i| { if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; @@ -95,14 +384,13 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduced_shape = reduced_shape.setDim(a, 1); } - const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); - const broad_op = dialects.stablehlo.broadcast_in_dim( + const broad_op = stablehlo.broadcast_in_dim( mlir_ctx, reduce_op.result(i), broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - tensor_type, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), + compiler.location, + ).appendTo(caller_scope.block); result[i] = Tensor._result(reduced_shape, broad_op.result(0)); } @@ -119,46 +407,52 @@ pub const ReduceWindowOpts = struct { padding: []const [2]i64, }; -pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); +pub fn ReduceWindowFn(N: comptime_int) type { + return @Fn( + &@as([N]type, @splat(ReduceArgs)), + &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), + [N]Tensor, + .{}, + ); +} - const mlir_ctx = CompilationContext.current().mlir_ctx; +pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func_: *const ReduceWindowFn(N)) [N]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; const reduce_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); - var args: ArgsType = undefined; - var block_types: [2 * inits.len]*const mlir.Type = undefined; + const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); + var args: Args = undefined; + var block_types: [2 * N]*const mlir.Type = undefined; - inline for (0..inits.len) |i| { + inline for (0..N) |i| { args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + N] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * N]*const mlir.Location = @splat(compiler.unknown_location); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - CompilationContext.current().pushBlock(reduce_block); - defer CompilationContext.current().popBlock(); + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); - const scope = CompilationContext.current().currentScope(); - inline for (0..inits.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; + inline for (0..N) |i| { + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + N); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args); - var result_values: [inits.len]*const mlir.Value = undefined; - inline for (0..inits.len) |i| { + var result_values: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -169,21 +463,12 @@ pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, com var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ &input_values, &init_values } }, - .result_type_inference = true, - .blocks = &.{reduce_block}, - .attributes = &.{ - .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), - .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), - .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), - .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), - // Cast the [][2]i64 to []i64 (safe) - .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + const reduce_op = stablehlo.reduce_window(mlir_ctx, &input_values, &init_values, reduce_block, opts.window_dimensions, .{ + .window_strides = opts.window_strides, + .base_dilations = opts.base_dilations, + .window_dilations = opts.window_dilations, + .padding = opts.padding, + }, compiler.location).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); @@ -197,14 +482,13 @@ pub const SortArgs = struct { right: Tensor, }; -pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const mlir_ctx = CompilationContext.current().mlir_ctx; +pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; const sort_block = b: { - const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); + const ArgsTypes: [inputs.len]type = @splat(SortArgs); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -212,26 +496,25 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); - block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); - block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); + block_types[2 * i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[2 * i + 1] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); const sort_block = mlir.Block.init(&block_types, &block_locs); errdefer sort_block.deinit(); - CompilationContext.current().pushBlock(sort_block); - defer CompilationContext.current().popBlock(); + const scope = compiler.pushBlock(sort_block); + defer scope.pop(); - const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; + scope.registerTensorAsBlockArgument(args[i].left.id, 2 * i); + scope.registerTensorAsBlockArgument(args[i].right.id, 2 * i + 1); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); - _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); + _ = stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); break :b sort_block; }; @@ -240,17 +523,10 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp input_values[i] = inputs[i].value(); } - const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ - .operands = .{ .variadic = &.{&input_values} }, - .result_type_inference = true, - .blocks = &.{sort_block}, - .attributes = &.{ - .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), - .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), - }, - .verify = true, - .location = .unknown(mlir_ctx), - }).appendTo(CompilationContext.current().currentScope().block); + const sort_op = stablehlo.sort(mlir_ctx, &input_values, sort_block, .{ + .dimension = axis_, + .is_stable = is_stable, + }, compiler.location).appendTo(compiler.currentScope().block); var result: [inputs.len]Tensor = undefined; inline for (0..inputs.len) |i| { @@ -260,20 +536,427 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytyp return result; } +/// Implement a while loop. +/// The given While struct contains all the parameters that are fixed through the while. +/// It must have: +/// * a State struct definition that captures all the loop state that will change on each iteration +/// * a `cond(While, State) Tensor` function returning a scalar boolean tensor +/// * a `body(While, State) State` function that preserves the state shapes. +pub fn @"while"( + While: type, + context: While, + initial_state: While.State, +) While.State { + const comp = Compiler.current(); + + var arena = std.heap.ArenaAllocator.init(comp.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const mlir_ctx = comp.mlir_ctx; + comp.pushLocation(@src(), @typeName(While)); + defer comp.popLocation(); + + // Force to materialize tensor.value() before we push a new scope. + var captured_context: While = undefined; + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, allocator, {}, context, &captured_context) catch unreachable; + + const flat_operands: []*const Tensor = meta.collectPtrs(Tensor, allocator, &initial_state) catch @panic("OOM"); + + const OperandInfo = struct { + type: *const mlir.Type, + location: *const mlir.Location, + value: *const mlir.Value, + }; + + var operands_info = std.MultiArrayList(OperandInfo).initCapacity(allocator, flat_operands.len) catch @panic("OOM"); + + for (flat_operands) |input| { + operands_info.appendAssumeCapacity(.{ + .type = mlirx.Type.rankedTensor(mlir_ctx, input._shape), + .location = comp.location, + .value = input.value(), + }); + } + + const cond_block = b: { + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); + errdefer block.deinit(); + + const scope = comp.pushBlock(block); + defer scope.pop(); + + comp.pushLocationFmt(@src(), "{s}.cond", .{@typeName(While)}); + defer comp.popLocation(); + + // Interpret initial_state as the block argument + for (0.., flat_operands) |i, input| { + scope.registerTensorAsBlockArgument(input.id, i); + } + + const cond: Tensor = captured_context.cond(initial_state); + stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); + + _ = stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); + break :b block; + }; + + const body_block, var result = b: { + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); + errdefer block.deinit(); + + const scope = comp.pushBlock(block); + defer scope.pop(); + + comp.pushLocationFmt(@src(), "{s}.body", .{@typeName(While)}); + defer comp.popLocation(); + + // Interpret operands as the block argument + scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); + for (0.., flat_operands) |i, input| { + scope.registerTensorAsBlockArgument(input.id, i); + } + + const result: While.State = captured_context.body(initial_state); + + if (builtin.mode == .debug) { + const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); + stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); + for (0.., flat_operands, result_tensors) |i, in, out| { + stdx.debug.assert(in.shape().eql(out.shape()), "zml.ops.while({s}) expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ @typeName(While), in, i, out }); + } + } + + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + _ = stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); + break :b .{ block, result }; + }; + + const op = stablehlo.while_( + mlir_ctx, + operands_info.items(.value), + operands_info.items(.type), + cond_block, + body_block, + comp.location, + ).appendTo(comp.currentScope().block); + + const AssignResultCtx = struct { + op_: *mlir.Operation, + og_tensors: []*const Tensor, + idx: usize = 0, + + fn assign(ctx: *@This(), tensor: *Tensor) void { + tensor.* = Tensor._result(ctx.og_tensors[ctx.idx].shape(), ctx.op_.result(ctx.idx)); + ctx.idx += 1; + } + }; + var assign_result_ctx: AssignResultCtx = .{ + .op_ = op, + .og_tensors = flat_operands, + }; + meta.visit(AssignResultCtx.assign, &assign_result_ctx, &result); + + return result; +} + +pub const while_ = @"while"; + +test @"while" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const While = struct { + // While struct can contain tensor and non-tensor fields. + limit: i64, + step: Tensor, + + pub const State = struct { + i: zml.Tensor, + sum: zml.Tensor, + }; + + pub fn cond(self: @This(), state: State) Tensor { + return state.i.cmp(.LT, Tensor.scalar(self.limit, .i64)); + } + + pub fn body(self: @This(), state: State) State { + return .{ + .i = state.i.add(self.step), + .sum = state.sum.add(self.step), + }; + } + + fn forward(i: Tensor, sum: Tensor) State { + return zml.ops.@"while"( + @This(), + .{ .limit = 10, .step = .scalar(1, .i64) }, + .{ .i = i, .sum = sum }, + ); + } + }; + + const initial_i: zml.Tensor = .init(.{}, .i64); + const initial_sum: zml.Tensor = .init(.{}, .i64); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + While.forward, + .{ initial_i, initial_sum }, + platform, + .{}, + ); + defer exe.deinit(); + + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); + defer i_buffer.deinit(); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); + defer sum_buffer.deinit(); + + var results: zml.Bufferized(While.State) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ i_buffer, sum_buffer }, .{&results}, .{ .wait = true }); + defer results.i.deinit(); + defer results.sum.deinit(); + + var cpu_i = try results.i.toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_i.free(std.testing.allocator); + var cpu_sum = try results.sum.toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_sum.free(std.testing.allocator); + + try std.testing.expectEqual(@as(i64, 10), cpu_i.items(i64)[0]); + try std.testing.expectEqual(@as(i64, 9), cpu_sum.items(i64)[0]); +} + +/// Implement if/else branches. +/// The given If struct contains all the parameters that can be referenced in the if/else branches. +/// +/// It must have: +/// * an `onTrue(If) T` function returning some tensors +/// * an `onFalse(If) T` function returning similar shaped tensors +/// +/// If some expression in `onTrue` or `onFalse` has side effects (like printing), +/// stablehlo guarantees they will only be evaluated in its branch. +pub fn @"if"( + If: type, + if_captures: If, + pred: Tensor, +) stdx.meta.FnReturn(If.onTrue) { + stdx.debug.assertComptime( + stdx.meta.FnReturn(If.onTrue) == stdx.meta.FnReturn(If.onFalse), + "zml.ops.if expects same return types for onTrue and onFalse, got {s} and {s}", + .{ @typeName(stdx.meta.FnReturn(If.onTrue)), @typeName(stdx.meta.FnReturn(If.onFalse)) }, + ); + + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); + + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + var blkctx: If = undefined; + + // Explicitly capture values from the parent block. + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; + + const mlir_ctx = Compiler.current().mlir_ctx; + const loc: *const mlir.Location = compiler.location; + + compiler.pushLocation(@src(), @typeName(If)); + defer compiler.popLocation(); + + const true_branch, const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); + + const result = blkctx.onTrue(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + + const false_branch, const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); + + const result = blkctx.onFalse(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + // TODO check that false branch values matches right branch + _ = false_branch; + + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); + _ = op.appendTo(Compiler.current().currentScope().block); + + return fromMlirOperationWithTags(op, true_branch); +} + +test "if" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + + const IfMod = struct { + a: Tensor, + b: Tensor, + + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return @"if"(@This(), .{ .a = a, .b = b }, pred.convert(.bool)); + } + + pub fn onTrue(ctx: @This()) Tensor { + return ctx.a.dotGeneral(ctx.b, &.{.{ 1, 0 }}, &.{}); + } + + pub fn onFalse(ctx: @This()) Tensor { + return ctx.b.dotGeneral(ctx.a, &.{.{ 1, 0 }}, &.{}); + } + }; + + { + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); + defer mod.deinit(); + } +} + +/// Simpler variant of `zml.ops.if` that assumes code-motion is supported by the backend. +/// The two branches are evaluated before the condition, then the condition chose which branches to keep. +pub fn if2( + pred: Tensor, + on_true: anytype, + on_false: @TypeOf(on_true), +) @TypeOf(on_true) { + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); + + var compiler = Compiler.current(); + + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + const mlir_ctx = compiler.mlir_ctx; + const loc: *const mlir.Location = compiler.location; + + const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); + defer allocator.free(true_values); + const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = compiler.pushBlock(block); + defer scope.pop(); + _ = stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); + break :b block; + }; + + const false_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_false) catch @panic("OOM"); + defer allocator.free(false_values); + const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = compiler.pushBlock(block); + defer scope.pop(); + + _ = stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); + break :b block; + }; + + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); + _ = op.appendTo(compiler.currentScope().block); + + return fromMlirOperationWithTags(op, on_true); +} + +test if2 { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + + const IfMod = struct { + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return if2( + pred.convert(.bool), + a.dotGeneral(b, &.{.{ 1, 0 }}, &.{}), + b.dotGeneral(a, &.{.{ 1, 0 }}, &.{}), + ); + } + }; + + { + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); + defer mod.deinit(); + } +} + +/// Create a Tensor struct similar to base, keeping base tags, +/// but using mlir value and dims from the mlir operation. +fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(base) { + const LocalContext = struct { + index: usize, + op: *const mlir.Operation, + }; + var context = LocalContext{ .index = 0, .op = op }; + var res = base; + meta.visit((struct { + fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { + var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); + stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); + // Copy tags; the partitioner infers result sharding. + // some ops can change dims eg reduceWindow, so we trust mlir here. + new._shape._tags = tensor._shape._tags; + tensor.* = new; + inner_ctx.index += 1; + } + }).cb, &context, &res); + std.debug.assert(context.index == op.numResults()); + return res; +} + pub const TritonOps = struct { debug: bool = false, - name: [:0]const u8, - ir: [:0]const u8, + name: []const u8, + ir: []const u8, grid: [3]i32, num_stages: i32, num_warps: i32, - output_operand_aliases: []const i64 = &.{}, + is_tma_allowed: bool = false, + global_scratch_memory_size: i32 = 0, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, }; /// Generate an MLIR call to the given member function with the given tensors. pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); var values: [inputs.len]*const mlir.Value = undefined; @@ -283,17 +966,19 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T var res_types: [outputs.len]*const mlir.Type = undefined; inline for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); } - const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ - mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), - mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), - mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), - mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), - mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), - mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), - mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + .named(mlir_ctx, "num_stages", .int(mlir_ctx, .i32, opts.num_stages)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "is_tma_allowed", .boolean(mlir_ctx, opts.is_tma_allowed)), + .named(mlir_ctx, "global_scratch_memory_size", .int(mlir_ctx, .i32, opts.global_scratch_memory_size)), }); var operands_layouts: [inputs.len][]const usize = undefined; @@ -306,7 +991,7 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -318,8 +1003,291 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T .result_layouts = &results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +pub const FlyOps = struct { + name: []const u8, + /// The whole module text: a `gpu.func @name ... kernel` inside a + /// `gpu.module`, one `!fly.ptr` per operand then per result; or a + /// top-level `func.func @name` in the destination-passing tensor ABI. + ir: []const u8, + grid: [3]i32, + /// Threads per block = num_warps * 64 on CDNA. + num_warps: i32, + waves_per_eu: i32 = 0, + /// Required when the kernel uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Indices into the full argument list, operands first, so result `i` is + /// `inputs.len + i`. A no-op inside HIP graphs: a kernel that must see + /// zeroed outputs should zero them itself. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + // The plugin silently ignores unknown or wrong-typed optional keys, so a + // misspelled one here is not an error, just an override that never lands. + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.waves_per_eu > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "waves_per_eu", .int(mlir_ctx, .i32, opts.waves_per_eu))); + if (opts.shared_mem_bytes > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "shared_mem_bytes", .int(mlir_ctx, .i64, opts.shared_mem_bytes))); + if (opts.zeroed_args.len > 0) { + const Opts = stablehlo.CustomCallOpts; + var zeroed: stdx.BoundedArray(*const mlir.Attribute, Opts.MAX_OPERANDS + Opts.MAX_RESULTS) = .empty; + for (opts.zeroed_args) |i| { + std.debug.assert(i >= 0 and i < inputs.len + outputs.len); + zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + } + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + // The kernel sees raw pointers and assumes row-major: pin the layouts. + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.fly", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "fly custom call, both entry ABIs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward` over one f32 input and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); + defer exe.deinit(); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); + defer buf.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // Native Fly ABI: a `gpu.func ... kernel` taking one !fly.ptr per operand + // then per result. This is what `zml.kernel.fly` emits. + { + const Mod = struct { + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @zml_fly_kernels { + \\ gpu.func @add_one(%a: !fly.ptr, %c: !fly.ptr) kernel { + \\ %tid = gpu.thread_id x + \\ %step = gpu.block_dim x + \\ %n = arith.constant 128 : index + \\ scf.for %i = %tid to %n step %step { + \\ %t = arith.index_cast %i : index to i32 + \\ %off = fly.make_int_tuple(%t) : (i32) -> !fly.int_tuple + \\ %pa = fly.add_offset(%a, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %pc = fly.add_offset(%c, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %va = fly.ptr.load(%pa) : (!fly.ptr) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %vc = arith.addf %va, %one : f32 + \\ fly.ptr.store(%vc, %pc) : (f32, !fly.ptr) -> () + \\ } + \\ gpu.return + \\ } + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 128 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i)) + 1, v); + } + + { + const Mod = struct { + const ir = + \\module { + \\ func.func @add_one(%input: tensor<64xf32>, %output: tensor<64xf32>) -> tensor<64xf32> { + \\ %thread = gpu.thread_id x + \\ %last = arith.constant 63 : index + \\ %i = arith.minui %thread, %last : index + \\ %value = tensor.extract %input[%i] : tensor<64xf32> + \\ %one = arith.constant 1.0 : f32 + \\ %sum = arith.addf %value, %one : f32 + \\ %updated = tensor.insert %sum into %output[%i] : tensor<64xf32> + \\ return %updated : tensor<64xf32> + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [64]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 64 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +pub const NeuronNkiOps = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + has_side_effect: bool = false, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// Generate an MLIR call for a Neuron NKI kernel. +/// +/// This API is Neuron-only: the source is compiled to an +/// `AwsNeuronCustomNativeKernel` backend config while emitting the graph. +pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs.len]Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => { + stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); + }, + } + if (comptime !platforms.isEnabled(.neuron)) unreachable; + const nki_kernel = @import("platforms/neuron/nki_kernel"); + + const mlir_ctx = ctx.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + var input_signatures: [inputs.len]nki_kernel.TensorSignature = undefined; + inline for (inputs, 0..) |input, i| { + input_signatures[i] = .{ + .dtype = @tagName(input.dtype()), + .dims = input.dims(), + }; + } + var output_signatures: [outputs.len]nki_kernel.TensorSignature = undefined; + inline for (outputs, 0..) |output, i| { + output_signatures[i] = .{ + .dtype = @tagName(output.dtype()), + .dims = output.dims(), + }; + } + + const compiled_backend_config = nki_kernel.compileNkiKernel(ctx.allocator, ctx.io, .{ + .name = opts.name, + .entrypoint = opts.entrypoint, + .source_path = opts.source_path, + .compiler_target = opts.compiler_target, + .inputs = &input_signatures, + .outputs = &output_signatures, + }) catch |err| { + stdx.debug.panic("failed to compile Neuron NKI kernel: {}", .{err}); + }; + defer ctx.allocator.free(compiled_backend_config); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "AwsNeuronCustomNativeKernel", + .has_side_effect = opts.has_side_effect, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + .additional_attributes = &.{ + .named(mlir_ctx, "backend_config", .string(mlir_ctx, compiled_backend_config)), + }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -333,11 +1301,11 @@ test "triton" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; + if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; const ir = \\ module { - \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { \\ %0 = tt.get_program_id x : i32 \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr @@ -366,7 +1334,7 @@ test "triton" { const a: zml.Tensor = .init(.{}, .f32); const b: zml.Tensor = .init(.{}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); @@ -374,7 +1342,10 @@ test "triton" { var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); - var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); + var results: [2]zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ a_buffer, b_buffer }, .{&results}, .{ .wait = true }); defer results[0].deinit(); defer results[1].deinit(); @@ -390,6 +1361,513 @@ test "triton" { try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); } +pub const CudaTileOps = struct { + name: []const u8, + ir: []const u8, + grid: [3]i32, + /// The Tile IR bytecode version XLA serializes at, "MAJOR.MINOR"; XLA's + /// default is 13.3. + ir_version: ?[]const u8 = null, + /// Result indices XLA zeroes before the launch, ascending. For a kernel + /// that accumulates, or writes less than the whole output. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.cuda_tile` custom call: XLA assembles `ir` with tileiras +/// and launches `name` over `grid` tile blocks. Every operand and result +/// arrives as one raw device pointer, in order, in the default layout. +pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cuda_tile")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.ir_version) |v| attrs.appendAssumeCapacity(.named(mlir_ctx, "ir_version", .string(mlir_ctx, v))); + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cuda_tile", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +pub const TclOps = struct { + ir: []const u8, + /// Outputs the function updates in place in an input's buffer. + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A TCL function the Furiosa plugin compiles as its own native unit. +pub fn tcl(inputs: anytype, outputs: anytype, opts: TclOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "furiosa.tcl", + .backend_config = .{ .original = opts.ir }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + +pub const MetalOps = struct { + /// The kernel symbol. + name: []const u8, + /// The kernel's module, in the AIR dialect. + ir: []const u8, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + /// What each `[[buffer(n)]]` binds: `k >= 0` the XLA buffer `k` (the + /// operands, then the results), `k < 0` the constant `-1 - k`. + arguments: []const i32, + /// The bytes of the constant arguments. + constants: []const []const u8 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.air` custom call: XLA lowers `ir` to AIR, compiles it to +/// a metallib and launches `name`. +pub fn metal(inputs: anytype, outputs: anytype, opts: MetalOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const allocator = compiler.allocator; + const constant_attrs = allocator.alloc(*const mlir.Attribute, opts.constants.len) catch @panic("OOM"); + defer allocator.free(constant_attrs); + for (opts.constants, constant_attrs) |bytes, *attribute| attribute.* = .denseArray(mlir_ctx, .i8, @as([]const i8, @ptrCast(bytes))); + + const sizes = [2][3]u32{ opts.grid, opts.threadgroup }; + var dims: [2][3]*const mlir.Attribute = undefined; + for (&dims, sizes) |*elems, dim| { + for (elems, dim) |*e, size| e.* = .int(mlir_ctx, .i32, size); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid", .array(mlir_ctx, &dims[0])), + .named(mlir_ctx, "threadgroup", .array(mlir_ctx, &dims[1])), + .named(mlir_ctx, "threadgroup_memory", .int(mlir_ctx, .i64, @as(i64, @intCast(opts.threadgroup_memory)))), + .named(mlir_ctx, "arguments", .denseArray(mlir_ctx, .i32, opts.arguments)), + .named(mlir_ctx, "constants", .array(mlir_ctx, constant_attrs)), + }); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "__gpu$xla.gpu.air", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + +pub const CuteOps = struct { + /// The kernel symbol. + name: []const u8, + /// Textual CuTe DSL module: either public `func.func` kernels, launched + /// with `grid`/`block`, or a `gpu.module` plus a host launch function + /// that owns the launch configuration. + ir: []const u8, + grid: ?[3]i32 = null, + block: ?[3]i32 = null, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?SymmetricWorkspace = null, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + + /// Named per-device workspace shared across CuTe calls and executables. + /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. + pub const SymmetricWorkspace = struct { + name: []const u8, + bytes: i64, + }; +}; + +/// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler +/// and launches `name`. Every operand and result arrives as one raw device +/// pointer, in order, in the default layout. +pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + }); + inline for (.{ "grid", "block" }) |key| { + if (@field(opts, key)) |dims| { + var elems: [3]*const mlir.Attribute = undefined; + for (&elems, dims) |*e, d| e.* = .int(mlir_ctx, .i32, d); + attrs.appendAssumeCapacity(.named(mlir_ctx, key, .array(mlir_ctx, &elems))); + } + } + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + if (opts.scalars.len > 0) { + const allocator = Compiler.current().allocator; + const scalar_attrs = allocator.alloc(*const mlir.Attribute, opts.scalars.len) catch @panic("OOM"); + defer allocator.free(scalar_attrs); + for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); + attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); + } + if (opts.symmetric_workspace) |workspace| { + stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); + attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), + .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), + }))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cute", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "cute" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32, in the CuTe DSL's frontend form. + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @kernels { + \\ cuda.kernel @add_one(%arg0: !cute.ptr>, %arg1: !cute.ptr>) attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array} { + \\ %i = nvvm.read.ptx.sreg.tid.x : i32 + \\ %shape = cute.make_shape() : () -> !cute.shape<"128"> + \\ %lay = cute.make_layout(%shape) : !cute.layout<"128:1"> + \\ %coord = cute.make_coord(%i) : (i32) -> !cute.coord<"?"> + \\ %in = cute.make_view(%arg0, %lay) : !cute.memref, "128:1"> + \\ %out = cute.make_view(%arg1, %lay) : !cute.memref, "128:1"> + \\ %x = cute.memref.load(%in, %coord) : (!cute.memref, "128:1">, !cute.coord<"?">) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %y = arith.addf %x, %one : f32 + \\ cute.memref.store(%out, %coord, %y) : (!cute.memref, "128:1">, !cute.coord<"?">, f32) -> () + \\ return + \\ } + \\ } + \\ func.func @launch(%arg0: !cute.ptr>, %arg1: !cute.ptr>) -> i32 attributes {llvm.emit_c_interface} { + \\ %smem = cute.kernel_smem_size @kernels::@add_one : i64 + \\ %c0_i64 = arith.constant 0 : i64 + \\ %stream = cuda.cast %c0_i64 : i64 -> !cuda.stream + \\ %c1 = arith.constant 1 : i32 + \\ %c128 = arith.constant 128 : i32 + \\ %cfg = cuda.launch_cfg.create (blockDim = (%c128, %c1, %c1), dynamicSmemBytes = %smem, gridDim = (%c1, %c1, %c1), stream = %stream) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + \\ %r = cuda.launch_ex @kernels::@add_one<%cfg> (%arg0, %arg1) : !cuda.launch_cfg, (!cute.ptr>, !cute.ptr>) -> !cuda.result + \\ %status = cuda.cast %r : !cuda.result -> i32 + \\ return %status : i32 + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cute(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +test "cuda_tile" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32; the XLA milestone kernel. + const ir = + \\cuda_tile.module @m { + \\ entry @add_one(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<128xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<128xptr> + \\ %in_p = offset %in_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<128xptr> -> tile<128xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<128xf32> + \\ %sum = addf %v, %one_b rounding : tile<128xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %t1 = store_ptr_tko weak %out_p, %sum : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "add_one", + .ir = ir, + .grid = .{ 1, 1, 1 }, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +test "cuda_tile grid" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Each of the 2x3x4 tile blocks writes its linear id t = x + 2y + 6z + // over out[128*t .. 128*t+128); the input is unused. + const ir = + \\cuda_tile.module @m { + \\ entry @grid(%in : tile>, %out : tile>) { + \\ %bx, %by, %bz = get_tile_block_id : tile + \\ %c2 = constant : tile + \\ %c6 = constant : tile + \\ %c128 = constant : tile + \\ %y2 = muli %by, %c2 : tile + \\ %z6 = muli %bz, %c6 : tile + \\ %xy = addi %bx, %y2 : tile + \\ %t = addi %xy, %z6 : tile + \\ %base = muli %t, %c128 : tile + \\ %offsets = iota : tile<128xi32> + \\ %base_r = reshape %base : tile -> tile<1xi32> + \\ %base_b = broadcast %base_r : tile<1xi32> -> tile<128xi32> + \\ %idx = addi %offsets, %base_b : tile<128xi32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %idx : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %tf = itof %t signed rounding : tile -> tile + \\ %tf_r = reshape %tf : tile -> tile<1xf32> + \\ %tf_b = broadcast %tf_r : tile<1xf32> -> tile<128xf32> + \\ %tok = store_ptr_tko weak %out_p, %tf_b : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "grid", + .ir = ir, + .grid = .{ 2, 3, 4 }, + })[0]; + } + }; + + const n = 24 * 128; + const a: zml.Tensor = .init(.{ .n = n }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + const input: [n]f32 = @splat(0); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i / 128)), v); + } +} + +test "cuda_tile zeroed_outputs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Writes only the first 64 of 128; `zeroed_outputs` owns the rest. + const ir = + \\cuda_tile.module @m { + \\ entry @half(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<64xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<64xptr> + \\ %in_p = offset %in_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<64xptr> -> tile<64xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<64xf32> + \\ %sum = addf %v, %one_b rounding : tile<64xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<64xptr> + \\ %out_p = offset %out_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %tok = store_ptr_tko weak %out_p, %sum : tile<64xptr>, tile<64xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "half", + .ir = ir, + .grid = .{ 1, 1, 1 }, + .zeroed_outputs = &.{0}, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + const want: f32 = if (i < 64) @floatFromInt(i + 1) else 0; + try std.testing.expectEqual(want, v); + } +} + pub const ScatterArgs = struct { input: Tensor, update: Tensor, @@ -408,17 +1886,19 @@ pub fn scatter( inputs: anytype, index_tensors: anytype, updates: anytype, - comptime func: anytype, + comptime func_: anytype, context: anytype, opts: Tensor.ScatterOpts, -) stdx.meta.FnResult(func) { - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); +) stdx.meta.FnReturn(func_) { + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); defer arena.deinit(); - const mlir_ctx = CompilationContext.current().mlir_ctx; + const mlir_ctx = compiler.mlir_ctx; const update_block, var result = b: { - const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); + const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); + const ArgsType = @Tuple(&ArgsTypes); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -426,31 +1906,30 @@ pub fn scatter( args[i].input = Tensor.init(.{}, inputs[i].dtype()); args[i].update = Tensor.init(.{}, inputs[i].dtype()); - block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); - block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].input.shape()); + block_types[i + inputs.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].update.shape()); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); const update_block = mlir.Block.init(&block_types, &block_locs); errdefer update_block.deinit(); - CompilationContext.current().pushBlock(update_block); - defer CompilationContext.current().popBlock(); + const scope = compiler.pushBlock(update_block); + defer scope.pop(); - const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; - scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; + scope.registerTensorAsBlockArgument(args[i].input.id, i); + scope.registerTensorAsBlockArgument(args[i].update.id, i + inputs.len); } - var result = @call(.auto, func, args ++ context); + var result = @call(.auto, func_, args ++ context); var result_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inputs.len) |i| { result_values[i] = result[i].value(); } - _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); break :b .{ update_block, result }; }; @@ -488,6 +1967,9 @@ pub fn scatter( return .{self.dynamicUpdateSlice(index_tensors, update)}; } + var updates_values: [inputs.len]*const mlir.Value = undefined; + LoweringCompatibility.preserveScatterDropSemantics(indices_per_axis.slice(), updates, opts, &updates_values); + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); @@ -499,10 +1981,7 @@ pub fn scatter( input_values[i] = inputs[i].value(); } - var updates_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); - - const op = dialects.stablehlo.scatter( + const op = stablehlo.scatter( mlir_ctx, &input_values, &.{indices.value()}, @@ -518,8 +1997,8 @@ pub fn scatter( .indices_are_sorted = opts.indices_are_sorted, .unique_indices = opts.indices_are_unique, }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor._result(inputs[i].shape(), op.result(i)); @@ -529,11 +2008,11 @@ pub fn scatter( } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, - indices_batch_axes: Shape.DimsArray = .{}, - scatter_to_operand_axes: Shape.DimsArray = .{}, - updates_transpose: Shape.AxesArray = .{}, + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + indices_batch_axes: Shape.DimsArray = .empty, + scatter_to_operand_axes: Shape.DimsArray = .empty, + updates_transpose: Shape.AxesArray = .empty, }; const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; @@ -544,11 +2023,11 @@ fn scatterConfig( indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - var scatter_to_operand_axes: Shape.DimsArray = .{}; - var updates_transpose: Shape.AxesArray = .{}; + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var indices_batch_axes: Shape.DimsArray = .empty; + var scatter_to_operand_axes: Shape.DimsArray = .empty; + var updates_transpose: Shape.AxesArray = .empty; const tagged_api = indices_axes.len > 0; const indices = indices_per_axis.get(0).shape(); @@ -631,14 +2110,14 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -694,12 +2173,12 @@ fn scatterPrepareIndices( cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); } - cfg.indices_batch_axes = .{}; + cfg.indices_batch_axes = .empty; // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; - var scatter_to_op_axes: Shape.DimsArray = .{}; + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .empty; + var scatter_to_op_axes: Shape.DimsArray = .empty; while (old_scatter_to_op_axes.len > 0) { const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; @@ -725,7 +2204,8 @@ pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; pub const GatherOpts = struct { indices_are_sorted: bool = false }; pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - const mlir_ctx = CompilationContext.current().mlir_ctx; + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -733,22 +2213,26 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } - var indices_shape = idx_per_axis[0].shape(); - for (idx_per_axis[1..]) |idx| { + var indices_per_axis, _ = Shape.parseStruct(Tensor, idx_per_axis); + + var indices_shape = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |idx| { if (idx.rank() > indices_shape.rank()) { indices_shape = idx.shape(); } } - for (idx_per_axis) |idx| { + for (indices_per_axis.slice()) |*idx| { stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + idx.* = idx.broad(indices_shape); } + LoweringCompatibility.preserveGatherFillSemantics(indices_per_axis.slice()); - var idx_batch_axes: Shape.DimsArray = .{}; + var idx_batch_axes: Shape.DimsArray = .empty; var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. @@ -766,13 +2250,13 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .empty; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); // Batching axis is already in self. if (is_batching_axis) continue; @@ -793,10 +2277,15 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // Sometimes the backend recognize this pattern, but not always. // So let us handle that. if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); + return self + .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) + // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise + // may hoist an all-gather before the slice and materialize the full tensor. + .optimizationBarrier() + .reshape(res_shape); } - var slice_dims: Shape.DimsArray = .{}; + var slice_dims: Shape.DimsArray = .empty; for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, @@ -806,9 +2295,9 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, } // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(idx_per_axis, .last, .coord); + const indices = Tensor.stack(indices_per_axis.constSlice(), .last, .coord); // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = dialects.stablehlo.gather( + const gather_op = stablehlo.gather( mlir_ctx, self.value(), indices.value(), @@ -822,16 +2311,127 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, - .unknown(mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + compiler.location, + ).appendTo(compiler.currentScope().block); const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return Tensor._result(res_shape, gather_op.result(0)); } +/// Backend lowering compatibility shims live here so public tensor/model code +/// can keep expressing StableHLO semantics directly. +/// Each rewrite should +/// preserve the source operation's semantics while avoiding a backend compiler +/// or runtime path that currently mishandles that legal StableHLO. +/// Fill/drop indexed ops use max-value integer sentinels for inactive lanes; +/// those lanes must not issue indirect memory accesses after lowering. +pub const LoweringCompatibility = struct { + /// Keep scalar integer broadcasts as scalar broadcasts until StableHLO broadcast emission. + /// + /// Some backend compilers fold rank-0 integer scalars into full-rank dense + /// splats too early, which can perturb later indexed-update lowering. A + /// scalar optimization barrier preserves the source-level broadcast shape. + pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { + switch (Compiler.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return null, + } + + if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; + return self.optimizationBarrier().broadcast(output_shape, &.{}); + } + + /// Preserve gather fill semantics before backend indirect-memory lowering. + /// + /// Frontends represent inactive gather rows by passing an out-of-bounds + /// integer sentinel together with fill semantics. StableHLO says those rows + /// should produce the fill value, so the sentinel lane is not a real memory + /// access. Some backend lowerings still route every lane through a gather/DGE + /// path before applying fill behavior, so sanitize inactive lanes early while + /// keeping the active rows unchanged. Related upstream Neuron report: + /// https://github.com/aws-neuron/aws-neuron-sdk/issues/1335. + pub fn preserveGatherFillSemantics(indices: []Tensor) void { + switch (Compiler.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return, + } + + const active_lanes = activeLanesForFillDropIndices(indices) orelse return; + replaceInactiveIndirectIndices(indices, active_lanes); + } + + /// Preserve scatter drop semantics before backend indirect-memory lowering. + /// Same as above but for scatter drop semantics. + pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { + const active_lanes: ?Tensor = switch (Compiler.current().platform.target) { + .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => null, + }; + + if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); + + inline for (updates, 0..) |update, i| { + const update_tensor = if (active_lanes) |active| blk: { + const mask = active.broad(update.shape().withDtype(.bool)); + const zero = Tensor.constant(update.dtype().zero()).broad(update.shape()); + break :blk mask.select(update, zero); + } else update; + update_values[i] = update_tensor.value(); + } + } + + /// Builds the lane mask used by fill/drop compatibility rewrites. + /// + /// For normal index widths, the max representable value is the inactive + /// sentinel. For 64-bit indices, the affected lowering path narrows to the + /// i32 indirect-index domain, so only values that remain valid after that + /// narrowing are considered active. When multiple index tensors form a + /// coordinate, a lane is active only if every component is active. + fn activeLanesForFillDropIndices(indices: []const Tensor) ?Tensor { + var active: ?Tensor = null; + for (indices) |idx| { + if (!idx.dtype().isInteger()) continue; + + const idx_active = switch (idx.dtype()) { + .i64 => blk: { + const zero = Tensor.scalar(@as(i64, 0), .i64).broad(idx.shape()); + const max_i32 = Tensor.scalar(@as(i64, std.math.maxInt(i32)), .i64).broad(idx.shape()); + break :blk idx.cmp(.GE, zero).logical(.AND, idx.cmp(.LE, max_i32)); + }, + .u64 => blk: { + const max_i32 = Tensor.scalar(@as(u64, std.math.maxInt(i32)), .u64).broad(idx.shape()); + break :blk idx.cmp(.LE, max_i32); + }, + inline .i2, .i4, .i8, .i16, .i32, .u2, .u4, .u8, .u16, .u32 => blk: { + const sentinel = Tensor.constant(idx.dtype().maxValue()).broad(idx.shape()); + break :blk idx.cmp(.NE, sentinel); + }, + .bool, .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16, .f16, .f32, .f64, .c64, .c128 => unreachable, + }; + active = if (active) |current| current.logical(.AND, idx_active) else idx_active; + } + return active; + } + + /// Replaces inactive sentinel indices with a safe index before indirect lowering. + /// + /// The replacement index is intentionally zero: it is always in-bounds for + /// non-empty operands and the corresponding scatter update is masked away by + /// the caller. This keeps the lowered indirect operation from touching the + /// sentinel address while preserving fill/drop semantics at the tensor level. + fn replaceInactiveIndirectIndices(indices: []Tensor, active_lanes: Tensor) void { + for (indices) |*idx| { + if (!idx.dtype().isInteger()) continue; + + const safe_index = Tensor.constant(idx.dtype().zero()).broad(idx.shape()); + idx.* = active_lanes.select(idx.*, safe_index); + } + } +}; + pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (bounded_array.constSlice(), 0..) |v, ax| { if (v == value) { res.appendAssumeCapacity(@intCast(ax)); @@ -840,16 +2440,17 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_R return res; } -fn TensorOrTensorArray(comptime T: type) type { - const type_info = @typeInfo(T); +fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { + const type_info = @typeInfo(OutputSpecT); return switch (type_info) { + .void => void, .@"struct" => |struct_info| b: { - if (T == Tensor) break :b Tensor; - if (!struct_info.is_tuple) @compileError("Expected tuple"); - break :b if (struct_info.fields.len == 1) + if (OutputSpecT == Shape) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); + break :b if (struct_info.field_names.len == 1) Tensor else - [struct_info.fields.len]Tensor; + [struct_info.field_names.len]Tensor; }, .array => |array_info| b: { break :b if (array_info.len == 1) @@ -858,19 +2459,156 @@ fn TensorOrTensorArray(comptime T: type) type { [array_info.len]Tensor; }, .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected slice"); + if (pointer_info.size != .slice) @compileError("Expected output shape slice"); break :b []Tensor; }, - else => @compileError("Unsupported type: " ++ @typeName(T)), + else => @compileError("Unsupported customCall output shape spec type: " ++ @typeName(OutputSpecT)), }; } pub const CustomCallOptions = struct { has_side_effect: bool, - output_operand_aliases: ?[]const i64 = null, + output_operand_aliases: ?[]const stablehlo.CustomCallOpts.OutputOperandAlias = null, + compute_on_host: bool = false, }; -pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { +fn customCallAdditionalAttributes(ctx: *Compiler, opts: CustomCallOptions) []const mlir.NamedAttribute { + if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; + + const frontend_attributes = mlir.Attribute.dict(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_compute_type", .string(ctx.mlir_ctx, "host")), + }); + + const additional_attributes = ctx.arena.allocator().alloc(mlir.NamedAttribute, 1) catch unreachable; + additional_attributes[0] = .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes); + return additional_attributes; +} + +pub fn customCallOutputOperandAliases( + comptime I: type, + comptime O: type, + comptime aliases: ?CustomCallOutputOperandAliases(I, O), +) ?[]const stablehlo.CustomCallOpts.OutputOperandAlias { + const aliases_ = aliases orelse return null; + + const output_field_names = @typeInfo(O).@"struct".field_names; + comptime var num_aliases = 0; + inline for (output_field_names) |field_name| { + if (@field(aliases_, field_name) != null) num_aliases += 1; + } + + const output_operand_aliases = comptime blk: { + var aliases_buffer: [num_aliases]stablehlo.CustomCallOpts.OutputOperandAlias = undefined; + var i = 0; + for (output_field_names, 0..) |field_name, output_index| { + if (@field(aliases_, field_name)) |operand| { + aliases_buffer[i] = .{ + .output_index = @intCast(output_index), + .operand_index = @intCast(@backingInt(operand)), + }; + i += 1; + } + } + break :blk aliases_buffer; + }; + + return &output_operand_aliases; +} + +pub const CompositeOpts = struct { + version: i32 = 0, + composite_attributes: []const mlir.NamedAttribute = &.{}, +}; + +pub fn composite( + name: [:0]const u8, + inputs: []const Tensor, + outputs: []const Shape, + comptime decomposition: anytype, + context: anytype, + opts: CompositeOpts, +) []Tensor { + const ctx = Compiler.current(); + const mlir_ctx = ctx.mlir_ctx; + + const decomp_name = ctx.allocPrint("{s}.impl_{d}", .{ name, ctx.nextCompositeId() }); + + { + const block_types = ctx.alloc(*const mlir.Type, inputs.len); + const block_locs = ctx.alloc(*const mlir.Location, inputs.len); + @memset(block_locs, ctx.unknown_location); + + for (inputs, 0..) |t, i| { + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, t.shape()); + } + + const block = mlir.Block.init(block_types, block_locs); + + const scope = ctx.pushBlock(block); + { + const arg_tensors = ctx.alloc(Tensor, inputs.len); + for (inputs, 0..) |t, i| { + arg_tensors[i] = Tensor._result(t.shape(), block.argument(i)); + } + + var result = @call(.auto, decomposition, .{ arg_tensors, context }); + const rtensors = ctx.alloc(Tensor, outputs.len); + meta.collectBuf((struct { + pub fn func_(t: Tensor) Tensor { + return t; + } + }).func_, {}, &result, rtensors); + + const rvals = ctx.alloc(*const mlir.Value, outputs.len); + for (rtensors, 0..) |t, i| { + rvals[i] = t.value(); + } + + _ = func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); + } + scope.pop(); + + _ = func.func(mlir_ctx, .{ + .name = decomp_name, + .block = block, + .location = ctx.location, + .visibility = .private, + .verify = false, + }).appendTo(ctx.module.body()); + } + + const operand_values = ctx.alloc(*const mlir.Value, inputs.len); + for (inputs, 0..) |t, i| { + operand_values[i] = t.value(); + } + + const result_types = ctx.alloc(*const mlir.Type, outputs.len); + for (outputs, 0..) |s, i| { + result_types[i] = mlirx.Type.rankedTensor(mlir_ctx, s); + } + + const op = stablehlo.composite( + mlir_ctx, + operand_values, + result_types, + .{ + .name = name, + .decomposition = decomp_name, + .composite_attributes = opts.composite_attributes, + .version = opts.version, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + + const out_tensors = ctx.alloc(Tensor, outputs.len); + for (outputs, 0..) |s, i| { + out_tensors[i] = Tensor._result(s, op.result(i)); + } + + return out_tensors; +} + +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { .@"struct" => |struct_info| b: { @@ -878,12 +2616,12 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Tensor{inputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.fields.len]Tensor = undefined; + var inputs_: [struct_info.field_names.len]Tensor = undefined; meta.collectBuf((struct { - pub fn func(t: Tensor) Tensor { + pub fn func_(t: Tensor) Tensor { return t; } - }).func, {}, &inputs, &inputs_); + }).func_, {}, &inputs, &inputs_); break :b &inputs_; }, .array => &inputs, @@ -896,17 +2634,18 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, // Transform generic outputs to flat slice. const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { + .void => &.{}, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) { break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.fields.len]Shape = undefined; + var output_shapes: [struct_info.field_names.len]Shape = undefined; meta.collectBuf((struct { - pub fn func(t: Shape) Shape { + pub fn func_(t: Shape) Shape { return t; } - }).func, {}, &outputs, &output_shapes); + }).func_, {}, &outputs, &output_shapes); break :b &output_shapes; }, .array => &outputs, @@ -917,21 +2656,31 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), }; - const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); + const outputs_flat = typedCustomCall( + target_name, + opts, + inputs_, + output_shapes, + metadata, + ); + if (comptime CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) == void) { + stdx.debug.assert(outputs_flat.len == 0, "customCall expected zero outputs for void return, got {}", .{outputs_flat.len}); + return; + } // Transform flat slice to generic outputs. return switch (@typeInfo(@TypeOf(outputs))) { .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.fields.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.fields.len]Tensor = undefined; + if (struct_info.field_names.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.field_names.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, .array => |array_info| b: { if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.fields.len]Tensor = undefined; + var outputs_: [array_info.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, @@ -943,123 +2692,358 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { - const ctx = CompilationContext.current(); - var arena = std.heap.ArenaAllocator.init(ctx.allocator); - defer arena.deinit(); +fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { + return switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => .int(mlir_ctx, .i8, value), + 16 => .int(mlir_ctx, .i16, value), + 32 => .int(mlir_ctx, .i32, value), + 64 => .int(mlir_ctx, .i64, value), + else => @panic("Unsupported DataType"), + }, + .unsigned => switch (int_field.bits) { + 8 => .int(mlir_ctx, .u8, value), + 16 => .int(mlir_ctx, .u16, value), + 32 => .int(mlir_ctx, .u32, value), + 64 => .int(mlir_ctx, .u64, value), + else => @panic("Unsupported DataType"), + }, + }; +} - const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; - for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } +fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.builtin.Type.Float, value: anytype) *const mlir.Attribute { + return switch (float_field.bits) { + 16 => .float(mlir_ctx, .f16, @as(f64, value)), + 32 => .float(mlir_ctx, .f32, @as(f64, value)), + 64 => .float(mlir_ctx, .f64, @as(f64, value)), + else => @panic("Unsupported DataType"), + }; +} - const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); - } +fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value: anytype) ?*const mlir.Attribute { + const type_info = @typeInfo(T); + return switch (type_info) { + .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), + .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), + else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), + }, + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), + .bool => .boolean(mlir_ctx, value), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + .string(mlir_ctx, value) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, + .optional => |optional_info| if (value) |wrapped_value| switch (@typeInfo(optional_info.child)) { + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, wrapped_value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, wrapped_value), + .bool => .boolean(mlir_ctx, wrapped_value), + else => @panic("Unsupported optional child type in metadata"), + } else null, + else => @compileError("Unsupported metadata type: " ++ @typeName(T)), + }; +} - const metadata_type_info = @typeInfo(@TypeOf(metadata)); - var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; - inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { - const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { - .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(metadata, field.name))), - .int => |int_field| switch (int_field.signedness) { - .signed => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(metadata, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(metadata, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(metadata, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(metadata, field.name)), - else => @panic("Unsupported DataType"), +pub fn CustomCall( + // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } + I: type, + O: type, + A: type, + // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O + // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` + // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error + comptime func_: anytype, + comptime params: struct { + /// Name of the custom call, must be unique. + name: [:0]const u8, + /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. + has_side_effect: bool, + /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. + /// Use the input/output field names to express the mapping: `{ . = . }` + output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, + /// Request XLA host-compute offload for this custom call on accelerator backends. + /// Input/Output buffers will automatically be moved d2h/h2d without the need for an explicit `.toMemory(...)`. + compute_on_host: bool = false, + }, +) type { + return struct { + pub const InputTensors = I; + pub const OutputShapes = O; + pub const Attributes = A; + + /// Register the custom call so you can use it. + pub fn register(platform: *const Platform) !void { + try platform.registerFfi(.{ + .name = params.name, + .platform_name = if (params.compute_on_host) "Host" else null, + .handler = @This().handler, + .traits = .{ + .command_buffer_compatible = true, }, - .unsigned => switch (int_field.bits) { - 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(metadata, field.name)), - 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(metadata, field.name)), - 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(metadata, field.name)), - 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(metadata, field.name)), - else => @panic("Unsupported DataType"), - }, - }, - .float => |float_field| switch (float_field.bits) { - 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), - 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), - 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), - else => @panic("Unsupported DataType"), - }, - .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(metadata, field.name)), - .pointer => |pointer_info| switch (pointer_info.size) { - .slice => if (pointer_info.child == u8) - mlir.stringAttribute(ctx.mlir_ctx, @field(metadata, field.name)) - else - @panic("Unsupported pointer type in metadata"), - else => @panic("Unsupported pointer type in metadata"), - }, - else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), - }; - metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); + }); + } + + pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { + const opts: CustomCallOptions = .{ + .has_side_effect = params.has_side_effect, + .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), + .compute_on_host = params.compute_on_host, + }; + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); + } + + fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); + return func_(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); + } + }; +} + +pub fn ShapeToTensor(T: type) type { + return meta.MapRestrict(Shape, Tensor).map(T); +} + +pub fn CustomCallOutputOperandAliases(I: type, O: type) type { + const field_names = @typeInfo(O).@"struct".field_names; + return @Struct( + .auto, + null, + field_names, + &@splat(?std.meta.FieldEnum(I)), + &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), + ); +} + +pub fn typedCustomCall( + target_name: [:0]const u8, + opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + + const ctx = Compiler.current(); + const allocator = ctx.arena.allocator(); + + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); + + const input_tensors: []const Tensor = switch (@typeInfo(Input)) { + .@"struct" => |struct_info| b: { + var input_tensors: [struct_info.field_names.len]Tensor = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + input_tensors[i] = @field(input, field_name); + } + break :b &input_tensors; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b input; + }, + else => @compileError("Unsupported input type: " ++ @typeName(Input)), + }; + + const output_shapes: []const Shape = switch (@typeInfo(Output)) { + .@"struct" => |struct_info| b: { + var output_shapes: [struct_info.field_names.len]Shape = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); + } + break :b &output_shapes; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b output; + }, + else => @compileError("Unsupported output type: " ++ @typeName(Output)), + }; + + var metadata_attribute_list = switch (@typeInfo(Attributes)) { + .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, + else => @compileError("Unsupported type: " ++ @typeName(Attributes)), + }; + if (@typeInfo(Attributes) == .@"struct") { + inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); + } + } + } + metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + }); + + const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; + const input_shapes = allocator.alloc(Shape, input_tensors.len) catch unreachable; + for (input_tensors, 0..) |input_tensor, i| { + values[i] = input_tensor.value(); + input_shapes[i] = input_tensor.shape(); } - const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - })); - - const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; - for (inputs, 0..) |input, i| { - operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_types[i] = .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); } - const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + const backend_config: *const mlir.Attribute = .dict(ctx.mlir_ctx, metadata_attribute_list.items); + + const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; + for (input_shapes, 0..) |shape, i| { + operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; } - const op = dialects.stablehlo.custom_call( + const result_layouts = allocator.alloc([]const usize, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + } + + const op = stablehlo.custom_call( ctx.mlir_ctx, values, - res_types, + result_types, .{ .call_target_name = target_name, .backend_config = .{ .typed_ffi = backend_config }, .has_side_effect = opts.has_side_effect, - .operand_layouts = operands_layouts, - .result_layouts = results_layouts, + .operand_layouts = operand_layouts, + .result_layouts = result_layouts, .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + .additional_attributes = customCallAdditionalAttributes(ctx, opts), }, - .unknown(ctx.mlir_ctx), - ).appendTo(CompilationContext.current().currentScope().block); + ctx.location, + ).appendTo(ctx.currentScope().block); - const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; - for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); + if (ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); } - return outputs_; + switch (@typeInfo(Output)) { + .@"struct" => |struct_info| { + var out: ShapeToTensor(Output) = undefined; + inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { + @field(out, field_name) = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + .pointer => |pointer_info| { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |output_shape, i| { + out[i] = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + else => @compileError("Unsupported output type: " ++ @typeName(Output)), + } +} + +fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { + TensorToCustomCallBuffer(I), + ShapeToCustomCallBuffer(O), + A, +} { + var input: TensorToCustomCallBuffer(I) = undefined; + inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input, field_name) = .fromPjrt(buf); + } + + var output: ShapeToCustomCallBuffer(O) = undefined; + inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { + const shape_buf = call_frame.results.buffers()[i]; + @field(output, field_name) = .fromPjrt(shape_buf); + } + + const attributes: A = switch (@typeInfo(A)) { + .void => {}, + .@"struct" => blk: { + var attrs: A = undefined; + inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(A)), + }; + + return .{ input, output, attributes }; } test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); - const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); - const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{0}, + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); } +test "CustomCallOutputOperandAliases maps named fields to indices" { + const Input = struct { + q: Tensor, + k: Tensor, + v: Tensor, + }; + const Output = struct { + attn: Shape, + scratch: Shape, + }; + + const aliases = comptime customCallOutputOperandAliases(Input, Output, .{ + .attn = .q, + .scratch = .v, + }).?; + + try std.testing.expectEqual(@as(usize, 2), aliases.len); + try std.testing.expectEqualDeep( + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, + aliases[0], + ); + try std.testing.expectEqualDeep( + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, + aliases[1], + ); +} + fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { - var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .empty; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig new file mode 100644 index 00000000..a0ec0037 --- /dev/null +++ b/zml/ops/manual_computation.zig @@ -0,0 +1,546 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. +/// +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. +/// +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` +/// +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 6cb4ca3b..824d2bd1 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,10 +1,13 @@ const std = @import("std"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects/stablehlo"); pub const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/pjrtx"); @@ -12,26 +15,21 @@ pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { - var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); - defer bytecode.deinit(); - module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { - log.err("failed to write module bytecode: {}", .{err}); - return switch (err) { - error.WriteFailed => error.OutOfMemory, - else => |e| e, - }; - }; - const stablehlo_version = blk: { if (api.stablehloCurrentVersion()) |requested_version| { - break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); + break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); } - break :blk dialects.stablehlo.minimumVersion(); + break :blk stablehlo.minimumVersion(); }; + const cloned_op = module.operation().clone(); + var cloned_module = mlir.Module.fromOperation(cloned_op); + defer cloned_module.deinit(); + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); - dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { + + stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { std.Io.Writer.Error.WriteFailed => error.OutOfMemory, @@ -47,6 +45,64 @@ pub const Client = opaque { } }; +pub fn TensorToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); +} + +pub fn ShapeToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Shape, CustomCallBuffer).map(T); +} + +pub const CustomCallBuffer = struct { + shape: Shape, + ptr: *anyopaque, + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } + + pub fn fromPjrt(buffer: *const pjrt.ffi.Buffer) @This() { + const dt = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return .{ + .ptr = buffer.data, + .shape = .init(buffer.dims(), dt), + }; + } + + // XLA: + // Align to 64-bytes, to mimic tsl::Allocator::kAllocatorAlignment. + // + // Preferred XLA:CPU alignment for buffers. XLA:CPU itself aligns intermediate + // buffers to this boundary (slices inside the preallocated temp buffer). + // https://github.com/openxla/xla/blob/fa2b7cfd909e8ffd343da1b7c5c341ace5e53aaa/xla/backends/cpu/alignment.h#L4 + pub const ALIGN = 64; + + // Assumes as host-side buffer + pub fn asHostSlice(self: CustomCallBuffer) []align(ALIGN) u8 { + const bytes: [*]align(ALIGN) u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostConstSlice(self: CustomCallBuffer) []align(ALIGN) const u8 { + const bytes: [*]align(ALIGN) const u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostScalar(self: CustomCallBuffer, T: type) T { + std.debug.assert(self.shape.byteSize() == @sizeOf(T)); + return std.mem.bytesAsValue(T, self.asHostConstSlice()[0..@sizeOf(T)]).*; + } +}; + pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), @@ -61,13 +117,13 @@ pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { } test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".fields) |field| { - const dt: DataType = @enumFromInt(field.value); + inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { + const dt: DataType = @fromBackingInt(@intCast(field_value)); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } - inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); + inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { + const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); if (dt == .invalid) continue; try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index daa2f9fe..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,38 +1,23 @@ const std = @import("std"); +const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms = @import("platforms"); -pub const Target = platforms.Platform; const stdx = @import("stdx"); +pub const Target = @import("platforms").Platform; +const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); +const profiler_ = @import("profiling/profiler.zig"); +const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); -pub const CompilationOptions = struct { - xla_dump_to: ?[]const u8 = null, - xla_dump_fusion_visualization: bool = false, - xla_dump_hlo_pass_re: ?[]const u8 = null, - sharding_enabled: bool = false, - sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, - device_memory_size: u64 = 0, -}; - -fn StaticPlatformMap(comptime E: type, comptime T: type) type { - const tag_count = @typeInfo(E).@"enum".fields.len; - var struct_field_names: [tag_count][]const u8 = undefined; - var struct_field_types: [tag_count]type = @splat(T); - const is_optional = @typeInfo(T) == .optional; - const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; - var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); - inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; - return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); -} - -var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; +var api_map: std.enums.EnumArray(Target, ?*const pjrt.Api) = .initFill(null); fn disableXlaLogs() void { // https://deepreg.readthedocs.io/en/latest/docs/logging.html#tensorflow-logging @@ -49,12 +34,27 @@ fn disableXlaLogs() void { ); } +fn validateDeviceCount(target: Target, num_devices: usize) !void { + if (num_devices == 0) { + log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); + return error.MissingDevices; + } + switch (target) { + .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { + if (!std.math.isPowerOfTwo(num_devices)) { + log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); + return error.InvalidDeviceCount; + } + }, + } +} + fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { - inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { + inline else => |tgt| api_map.get(tgt) orelse b: { disableXlaLogs(); - const api = try platforms.load(allocator, io, tag); - @field(api_map, @tagName(tag)) = api; + const api = try tgt.load(allocator, io); + api_map.set(tgt, api); break :b api; }, }; @@ -72,12 +72,9 @@ pub const Memory = struct { platform: *const Platform, addressable_by_devices: []*const Device, - fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform, all_devices: []const Device) !Memory { + fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform) !Memory { const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); - for (pjrt_addressable_by_devices, addressable_by_devices) |pjrt_device, *addressable_by_device| { - addressable_by_device.* = &all_devices[pjrt_device.localHardwareId(platform.pjrt_api)]; - } return .{ .pjrt_memory = pjrt_memory, @@ -92,7 +89,7 @@ pub const Memory = struct { pub fn isOfKind(self: Memory, kind_: Kind) bool { switch (self.platform.target) { - .cuda, .rocm => { + .cuda, .rocm, .oneapi, .tpu, .furiosa => { const zml_kind: Memory.Kind = switch (self.kind().len) { "device".len => .device, "pinned_host".len => .host_pinned, @@ -101,13 +98,20 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu, .tpu, .neuron => return true, + .cpu, .neuron, .metal => return true, } } fn deinit(self: *Memory, allocator: std.mem.Allocator) void { allocator.free(self.addressable_devices); } + + fn populateAddressableByDevices(self: *Memory) void { + const pjrt_addressable_by_devices = self.pjrt_memory.addressableByDevices(self.platform.pjrt_api); + for (pjrt_addressable_by_devices, self.addressable_by_devices) |pjrt_device, *addressable_by_device| { + addressable_by_device.* = self.platform.deviceFromPjrt(pjrt_device); + } + } }; pub const Device = struct { @@ -115,19 +119,30 @@ pub const Device = struct { pjrt_device: *const pjrt.Device, pjrt_desc: *const pjrt.DeviceDescription, addressable_memories: []*const Memory, + memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory), - fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform, all_addressable_memories: []const Memory) !Device { + fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform) !Device { const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { - addressable_memory.* = &all_addressable_memories[pjrt_memory.id(platform.pjrt_api)]; + addressable_memory.* = platform.memoryFromPjrt(pjrt_memory); } + // Cache memory lookups since they are expensive + const default_memory: *const Memory = resolveDefaultMemory(pjrt_device_, platform, addressable_memories); + const memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory) = .init(.{ + .default = default_memory, + .device = resolveMemory(addressable_memories, .device), + .host_pinned = resolveMemory(addressable_memories, .host_pinned), + .host_unpinned = resolveMemory(addressable_memories, .host_unpinned), + }); + return .{ .platform = platform, .pjrt_device = pjrt_device_, .pjrt_desc = pjrt_device_.getDescription(platform.pjrt_api), .addressable_memories = addressable_memories, + .memory_by_kind = memory_by_kind, }; } @@ -135,8 +150,27 @@ pub const Device = struct { allocator.free(self.addressable_memories); } - pub fn id(self: Device) i32 { - return self.pjrt_desc.id(self.platform.pjrt_api); + fn resolveDefaultMemory(pjrt_device_: *const pjrt.Device, platform: *const Platform, addressable_memories: []*const Memory) *const Memory { + const pjrt_memory = pjrt_device_.defaultMemory(platform.pjrt_api); + for (addressable_memories) |mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + return platform.memoryFromPjrt(pjrt_memory); + } + + fn resolveMemory(addressable_memories: []*const Memory, memory_kind: Memory.Kind) ?*const Memory { + std.debug.assert(memory_kind != .default); + + for (addressable_memories) |mem| { + if (mem.isOfKind(memory_kind)) { + return mem; + } + } + return null; + } + + pub fn id(self: Device) u32 { + return @intCast(self.pjrt_desc.id(self.platform.pjrt_api)); } pub fn processIndex(self: Device) i32 { @@ -144,7 +178,7 @@ pub const Device = struct { } pub fn localHardwareId(self: Device) i32 { - return self.device.localHardwareId(self.platform.pjrt_api); + return @intCast(self.pjrt_device.localHardwareId(self.platform.pjrt_api)); } pub fn kind(self: Device) []const u8 { @@ -163,6 +197,19 @@ pub const Device = struct { return self.pjrt_device.memoryStats(self.platform.pjrt_api) catch .zeroes; } + /// NUMA node closest to the device, when the plugin reports one. + pub fn numaNode(self: Device) ?u32 { + const attributes = self.pjrt_device.attributes(self.platform.pjrt_api) catch return null; + defer attributes.deinit(); + + const value = attributes.get("numa_node") orelse return null; + + return switch (value) { + .int64 => |node| std.math.cast(u32, node), + else => null, + }; + } + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} ({s})", .{ self.pjrt_desc.kind(self.platform.pjrt_api), @@ -170,36 +217,98 @@ pub const Device = struct { }); } - pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { - if (memory_kind == .default) { - const mem = self.pjrt_device.defaultMemory(self.platform.pjrt_api); - return &self.platform.memories[mem.id(self.platform.pjrt_api)]; - } + pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { + return self.memory_by_kind.values[@backingInt(memory_kind)]; + } +}; - for (self.addressable_memories) |mem| { - if (mem.isOfKind(memory_kind)) { - return mem; +fn platformDeviceSortId(target: Target, device: Device) usize { + return switch (target) { + .neuron => @intCast(device.localHardwareId()), + .cuda, .rocm, .tpu, .cpu, .oneapi, .metal, .furiosa => device.id(), + }; +} + +fn sortDevicesById(target: Target, devices: []Device) void { + const Context = struct { + target: Target, + + fn lessThan(ctx: @This(), lhs: Device, rhs: Device) bool { + return platformDeviceSortId(ctx.target, lhs) < platformDeviceSortId(ctx.target, rhs); + } + }; + + std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); + + if (builtin.mode == .debug) { + for (devices, 0..) |device, expected_id| { + std.debug.assert(platformDeviceSortId(target, device) == expected_id); + } + } +} + +// State union tagged on target platform to handle related resources +pub const State = union(Target) { + cpu: void, + cuda: CudaState, + rocm: void, + tpu: void, + neuron: void, + oneapi: void, + metal: void, + furiosa: void, + + pub const CudaState = struct { + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, + + fn deinit(self: *CudaState) void { + if (self.flashinfer_cutlass_moe_runners) |runners| { + runners.deinit(); + self.flashinfer_cutlass_moe_runners = null; } } - unreachable; + }; + + pub fn init(target: Target) State { + return switch (target) { + .cpu => .{ .cpu = {} }, + .cuda => .{ .cuda = .{} }, + .rocm => .{ .rocm = {} }, + .tpu => .{ .tpu = {} }, + .neuron => .{ .neuron = {} }, + .oneapi => .{ .oneapi = {} }, + .metal => .{ .metal = {} }, + .furiosa => .{ .furiosa = {} }, + }; + } + + pub fn deinit(self: *State) void { + switch (self.*) { + .cuda => |*cuda_state| cuda_state.deinit(), + else => {}, + } } }; pub const Platform = struct { - arena_state: std.heap.ArenaAllocator.State = .{}, + arena: std.heap.ArenaAllocator, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, + state: State, devices: []const Device, memories: []const Memory, + physical_mesh: zml.Sharding.PhysicalMesh, + meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), + replicated_mesh: *const zml.Mesh, + io_impl: IoImpl, - // This make the pjrt struct quite fat, but is only used during compilation. - // TODO: Reconsider having it here, and maybe pass explicitly to compile, - // or create an intermediary struct: - // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` - compilation_options: CompilationOptions = .{}, + pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; - pub const MAX_NUM_DEVICES: u8 = if (platforms.isEnabled(.tpu)) 32 else 8; + pub const IoImpl = enum { + threaded, + zio, + }; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); @@ -207,61 +316,89 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); const pjrt_devices = pjrt_client.addressableDevices(api); + try validateDeviceCount(target, pjrt_devices.len); if (pjrt_devices.len > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - const devices = try arena.allocator().alloc(Device, pjrt_devices.len); - const pjrt_memories = pjrt_client.addressableMemories(api); - const memories = try arena.allocator().alloc(Memory, pjrt_memories.len); - const platform = try arena.allocator().create(Platform); - platform.* = .{ - .target = target, - .pjrt_api = api, - .pjrt_client = pjrt_client, - .compilation_options = .{}, - .devices = devices, - .memories = memories, + // Note: Platform is a self-owning struct. It contains the arena that created it in the first place + // But it does mean we have to be careful to pass the arena state that contains the node + const platform: *Platform = platform: { + var initial_arena = std.heap.ArenaAllocator.init(allocator); + errdefer initial_arena.deinit(); + + var initial_allocator = initial_arena.allocator(); + // Pre-heat the arena, to avoid fragmentation. + initial_allocator.free(try initial_allocator.alloc(u8, 8 * 1024)); + + const platform = try initial_allocator.create(Platform); + platform.* = .{ + .arena = initial_arena, + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .state = State.init(target), + .meshes = .empty, + .replicated_mesh = undefined, + // set below + .devices = undefined, + .memories = undefined, + .physical_mesh = undefined, + .io_impl = options.io_impl, + }; + break :platform platform; }; - defer platform.arena_state = arena.state; + + const arena = platform.arena.allocator(); + errdefer platform.arena.deinit(); + try platform.meshes.ensureTotalCapacity(arena, 8); { - for (pjrt_devices, devices) |pjrt_device, *platform_device| { - platform_device.* = try .init(arena.allocator(), pjrt_device, platform, memories); - } + const devices = try arena.alloc(Device, pjrt_devices.len); + platform.devices = devices; + const memories = try arena.alloc(Memory, pjrt_memories.len); + platform.memories = memories; + + // TODO: part of the complication here is that we layout the data in spaghetti mode, + // where devices and memories point to each other and also point to the platform. for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { - platform_memory.* = try .init(arena.allocator(), pjrt_memory, platform, devices); + platform_memory.* = try .init(arena, pjrt_memory, platform); } + for (pjrt_devices, devices) |pjrt_device, *platform_device| { + platform_device.* = try .init(arena, pjrt_device, platform); + } + sortDevicesById(target, devices); + for (memories) |*platform_memory| { + platform_memory.populateAddressableByDevices(); + } + + platform.physical_mesh = try switch (options.physical_mesh) { + .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), + .custom => |builder| builder(arena, target, devices), + }; + platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } switch (target) { .cuda => { - zml.attention.flashattn.load(arena.allocator(), io) catch { - log.warn("Failed to load flashattn", .{}); - }; - zml.attention.flashattn.register(platform) catch { - log.warn("Failed to register flashattn custom call", .{}); + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); }; }, else => {}, } - if (platform.pjrt_api.ffi()) |ffi| b: { - const platform_name = switch (target) { - .cuda => "cuda", - .rocm => "rocm", - .cpu => "host", - else => break :b, - }; - ffi.register(platform.pjrt_api, "zml$print", platform_name, printCallback, .{ .command_buffer_compatible = false }) catch |e| { - log.warn("Failed to register \"print\" callback, error: {}", .{e}); - }; - } + platform.registerFfi( + .{ + .name = "zml$print", + .handler = printCallback, + .traits = .{ .command_buffer_compatible = false }, + }, + ) catch |err| { + log.warn("Failed to register FFI custom call \"zml$print\", error: {}", .{err}); + }; return platform; } @@ -272,6 +409,9 @@ pub const Platform = struct { .neuron, .rocm, .cuda, + .oneapi, + .metal, + .furiosa, .cpu, }; return for (ordered_targets) |target| { @@ -279,14 +419,14 @@ pub const Platform = struct { } else error.Unavailable; } - pub fn formatWithDevices(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; const langle = "└─ "; try writer.print("platform: {s}\n", .{@tagName(self.target)}); try writer.print("version: {f}\n", .{self.pjrt_api.version()}); - try writer.print("stablehlo_version: {?s}\n", .{self.pjrt_api.stablehloCurrentVersion()}); + try writer.print("extensions:\n", .{}); { var it = self.pjrt_api.extensions(); @@ -295,42 +435,115 @@ pub const Platform = struct { } } + try writer.print("plugin attributes:\n", .{}); + { + const attributes = self.pjrt_api.pluginAttributes(); + if (attributes.len == 0) { + try writer.print("{s}(none)\n", .{langle}); + } else { + for (attributes, 0..) |attr, i| { + const is_last_attr = i == attributes.len - 1; + try writer.print("{s}{s}", .{ if (is_last_attr) langle else tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| { + for (v, 0..) |item, j| { + if (j == 0) { + try writer.print(": {d}", .{item}); + } else { + try writer.print(".{d}", .{item}); + } + } + }, + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + } + } + } + try writer.print("devices:\n", .{}); for (self.devices, 0..) |device, i| { - const is_last_device = i < self.devices.len - 1; - try writer.print("{s}{f}\n", .{ if (is_last_device) tee else langle, device }); - for (device.addressable_memories, 0..) |mem, j| { - try writer.print("{s}{s}memory: {s}\n", .{ - if (is_last_device) line else " ", - if (j < device.addressable_memories.len - 1) tee else langle, - mem.kind(), - }); + const is_last_device = i == self.devices.len - 1; + const child_indent = if (is_last_device) " " else line; + + try writer.print("{s}{f}\n", .{ if (is_last_device) langle else tee, device }); + + { + const device_attrs = device.pjrt_desc.attributes(self.pjrt_api); + if (device_attrs.len > 0) { + var last_name: ?[]const u8 = null; + var remaining: usize = device_attrs.len; + + while (remaining > 0) : (remaining -= 1) { + var next_index: ?usize = null; + for (device_attrs, 0..) |attr, j| { + const name = attr.name(); + if (last_name) |last| { + if (std.mem.order(u8, name, last) != .gt) continue; + } + if (next_index) |ni| { + if (std.mem.order(u8, name, device_attrs[ni].name()) == .lt) { + next_index = j; + } + } else { + next_index = j; + } + } + + if (next_index == null) break; + const attr = device_attrs[next_index.?]; + + try writer.print("{s}{s}{s}", .{ child_indent, tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| try writer.print(": {any}", .{v}), + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + + last_name = attr.name(); + } + } + } + + try writer.print("{s}{s}memories:\n", .{ child_indent, langle }); + { + const memory_indent = " "; + if (device.addressable_memories.len == 0) { + try writer.print("{s}{s}{s}(none)\n", .{ child_indent, memory_indent, langle }); + } else { + for (device.addressable_memories, 0..) |mem, j| { + const is_last_mem = j == device.addressable_memories.len - 1; + try writer.print("{s}{s}{s}memory: {s}\n", .{ + child_indent, + memory_indent, + if (is_last_mem) langle else tee, + mem.pjrt_memory.debugString(self.pjrt_api), + }); + } + } } } } - pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithDevices) { + pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithAttributes) { return .{ .data = self }; } - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; - - pub fn sharding(self: *const Platform) Sharding { - // replicas run the same function but with different inputs, - // while partitions contribute to one evaluation over a shared input. - // Inside an inference process, we generally don't want replicas, - // as it's best to fully isolate replicas on different processes. - // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.devices.len); - return if (self.compilation_options.sharding_enabled) - .{ .num_replicas = 1, .num_partitions = num_devices } - else - .{ .num_replicas = 1, .num_partitions = 1 }; - } - - pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { + pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { + _ = io; + _ = allocator; + if (comptime Target.cuda.isEnabled()) { + self.state.deinit(); + } + self.physical_mesh.deinit(self.arena.allocator()); self.pjrt_client.deinit(self.pjrt_api); - self.arena_state.promote(allocator).deinit(); + self.arena.deinit(); } pub fn compile( @@ -342,12 +555,14 @@ pub const Platform = struct { args: stdx.meta.Tail( std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), ), + opts: zml.Compiler.Options, ) !Exe { return self.compileFn( allocator, io, @field(@TypeOf(model_), @tagName(func)), .{model_} ++ args, + opts, ); } @@ -358,8 +573,9 @@ pub const Platform = struct { comptime func: anytype, model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), + opts: zml.Compiler.Options, ) !Exe { - return self.compileFn(allocator, io, func, .{model} ++ args); + return self.compileFn(allocator, io, func, .{model} ++ args, opts); } pub fn compileFn( @@ -368,8 +584,9 @@ pub const Platform = struct { io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), + opts: zml.Compiler.Options, ) !Exe { - return zml.module.compile(allocator, io, func, args, self); + return zml.Compiler.compile(allocator, io, self, func, args, opts); } pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { @@ -389,18 +606,138 @@ pub const Platform = struct { } unreachable; } + + pub const FfiRegistration = struct { + name: []const u8, + handler: *const pjrt.ffi.Handler, + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + platform_name: ?[]const u8 = null, + }; + + pub fn registerFfi(self: *const zml.Platform, registration: FfiRegistration) !void { + const platform_name = registration.platform_name orelse self.pjrt_client.platformName(self.pjrt_api); + if (self.pjrt_api.ffi()) |ffi| { + try ffi.register(self.pjrt_api, registration.name, platform_name, registration.handler, registration.traits); + } else { + log.warn("PJRT FFI extension not available for {s}", .{@tagName(self.target)}); + } + } + + pub const Profiler = profiler_.Profiler; + pub const ProfilerOptions = profiler_.ProfilerOptions; + + pub fn profiler(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + return try profiler_.profiler(self.pjrt_api, allocator, io, options); + } + + /// Create a Mesh based on the given logical mesh and the default strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { + return platform.registerMeshWithStrategy( + name, + logical, + .suggest(logical, &platform.physical_mesh), + ) catch |err| switch (err) { + error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis, error.OverlappingLogicalAxes => { + std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + }, + error.InvalidPartialBinding, error.InvalidLogicalDimension, error.IncompatibleLogicalDimension => { + std.debug.panic("ZML cannot satisfy the dimensions of logical mesh {f} on physical mesh {f}", .{ logical, platform.physical_mesh }); + }, + error.OutOfMemory => |e| return e, + }; + } + + /// Create a Mesh based on the given logical mesh and a strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { + const arena = platform.arena.allocator(); + const entry = try platform.meshes.getOrPut(arena, name); + if (entry.found_existing) { + std.debug.panic("Another mesh already exists with this name: {s}", .{name}); + } + + const owned_name = try arena.dupe(u8, name); + const owned_mesh = try arena.create(Sharding.Mesh); + owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); + entry.key_ptr.* = owned_name; + entry.value_ptr.* = owned_mesh; + + return owned_mesh; + } + + fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { + for (self.memories) |*mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + unreachable; + } + + fn deviceFromPjrt(self: *const Platform, pjrt_device: *const pjrt.Device) *const Device { + for (self.devices) |*device| { + if (device.pjrt_device == pjrt_device) return device; + } + unreachable; + } + + pub inline fn defaultMemoryLayout(platform: *const Platform, dims: []const i64, dtype: zml.DataType) pjrt.MemoryLayout { + // inline cause `default` is a huge ass struct allocated on the stack, + // and toMemoryLayout returns slices into it. + // There is probably a better way of doing this, + // The queried layout can include backend-specific tiling. + return switch (platform.target) { + .tpu, .furiosa => { + if (comptime !Target.tpu.isEnabled() and !Target.furiosa.isEnabled()) unreachable; + const element_type = pjrtx.bufferTypeFromDtype(dtype); + const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); + return default.toMemoryLayout(); + }, + .cuda, .rocm, .neuron, .oneapi, .cpu, .metal => .{ + // If this is the default layout on the platform, there is no point calling PJRT + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; + } }; pub const CreateOptions = struct { + pub const CreatePhysicalMeshFn = *const fn ( + allocator: std.mem.Allocator, + target: Target, + devices: []const Device, + ) anyerror!zml.Sharding.PhysicalMesh; + + pub const PhysicalMesh = union(enum) { + auto, + custom: CreatePhysicalMeshFn, + }; + + physical_mesh: PhysicalMesh = .auto, cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. - // Even on a 8GB GPU it should leave enough space for the Cuda driver + // Even on a 8GB GPU it should leave enough space for the platform driver/runtime. // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, - rocm: struct {} = .{}, + rocm: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, + cuda: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, tpu: struct {} = .{}, neuron: struct {} = .{}, + furiosa: Furiosa = .{}, + oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + io_impl: Platform.IoImpl = .threaded, + + pub const Furiosa = struct { + pe_count: u8 = 8, + + fn writeNamedValues(self: Furiosa, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(.init(.int64, "pe_count", self.pe_count)); + } + }; pub const Cpu = struct { device_count: u32, @@ -410,8 +747,11 @@ pub const CreateOptions = struct { } }; - pub const Cuda = struct { + pub const XlaGpu = struct { allocator: Allocator = .{ .bfc = .{} }, + /// The PJRT C API still exposes this under legacy + /// `use_tfrt_gpu_client` name. + gpu_async_dispatch: bool = false, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 // visible_devices: []const i64 = &.{}, // node_id @@ -424,6 +764,8 @@ pub const CreateOptions = struct { bfc: Options, /// use cudaMallocAsync async: Options, + /// use virtual memory allocator + vmm: VmmOptions, /// use raw cuMalloc platform, @@ -432,9 +774,13 @@ pub const CreateOptions = struct { memory_fraction: f32 = 0.90, collective_memory_size_mb: i64 = 0, }; + + pub const VmmOptions = struct { + memory_fraction: f32 = 0.90, + }; }; - fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { + fn writeNamedValues(self: XlaGpu, target: Target, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(.init(.string, "allocator", "platform")); @@ -443,7 +789,7 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.string, "allocator", switch (self.allocator) { .bfc => "bfc", .async => "cuda_async", - .platform => unreachable, + .platform, .vmm => unreachable, })); values.appendAssumeCapacity(.init(.bool, "preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { @@ -453,6 +799,16 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.int64, "collective_memory_size", opt.collective_memory_size_mb * 1024 * 1024)); } }, + .vmm => |opt| { + values.appendAssumeCapacity(.init(.string, "allocator", "vmm")); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); + } + }, + } + switch (target) { + .cuda => values.appendAssumeCapacity(.init(.bool, "use_tfrt_gpu_client", self.gpu_async_dispatch)), + else => {}, } } }; @@ -462,7 +818,11 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .cuda => self.cuda.writeNamedValues(&values), + .furiosa => self.furiosa.writeNamedValues(&values), + .cuda => self.cuda.writeNamedValues(target, &values), + .rocm => self.rocm.writeNamedValues(target, &values), + .oneapi => self.oneapi.writeNamedValues(target, &values), + .metal => self.metal.writeNamedValues(target, &values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); @@ -475,14 +835,136 @@ pub const CreateOptions = struct { // TODO(Corendos): Consider moving that in its own file if its size increase too much. pub const cuda = struct { - pub fn tryGetComputeCapabilities(platform: *const zml.Platform, device: *const pjrt.Device) ?[]const u8 { - stdx.debug.assert(platform.target == .cuda, "tryGetComputeCapabilities expects .cuda platform, got {}", .{platform.target}); + pub const ComputeCapability = struct { + major: u8, + minor: u8, + + pub fn parse(text: []const u8) ?ComputeCapability { + var parts = std.mem.splitScalar(u8, text, '.'); + return .{ + .major = std.fmt.parseInt(u8, parts.first(), 10) catch return null, + .minor = std.fmt.parseInt(u8, parts.next() orelse "0", 10) catch return null, + }; + } + + pub fn eql(self: ComputeCapability, other: ComputeCapability) bool { + return self.major == other.major and self.minor == other.minor; + } + + pub fn atLeast(self: ComputeCapability, other: ComputeCapability) bool { + return self.major > other.major or (self.major == other.major and self.minor >= other.minor); + } + + pub fn sm(self: ComputeCapability) u16 { + return @as(u16, self.major) * 10 + self.minor; + } + }; + + /// SM count of an addressable CUDA device; unavailable attributes return null. + pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { + const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; + return std.math.cast(u32, attribute.int64); + } + + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + if (platform.target != .cuda) return null; + const devices = platform.pjrt_client.devices(platform.pjrt_api); + if (devices.len == 0) return null; + + const attributes = devices[0].getDescription(platform.pjrt_api).attributes(platform.pjrt_api); + return for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), "compute_capability")) { + break ComputeCapability.parse(attr.value().string); + } + } else null; + } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } +}; + +pub const rocm = struct { + pub const ComputeCapability = enum { + /// CDNA1: MI100. + gfx908, + /// CDNA2: MI200 series. + gfx90a, + /// CDNA3: MI300 series. + gfx942, + /// CDNA4: MI350 series. + gfx950, + /// RDNA2: RX 6000 series. + gfx1030, + /// RDNA3 (Navi 31): RX 7900 series. + gfx1100, + /// RDNA3 (Navi 32): RX 7800 and RX 7700 series. + gfx1101, + /// RDNA3 (Navi 33): RX 7600 series. + gfx1102, + /// RDNA3 APU: Phoenix. + gfx1103, + /// RDNA3.5: newer Ryzen AI APUs. + gfx1150, + /// RDNA4 (Navi 44): RX 9060 family. + gfx1200, + /// RDNA4 (Navi 48): RX 9070 family. + gfx1201, + + pub const Architecture = enum { + cdna1, + cdna2, + cdna3, + cdna4, + rdna2, + rdna3, + rdna3_5, + rdna4, + }; + + pub fn architecture(self: ComputeCapability) Architecture { + return switch (self) { + .gfx908 => .cdna1, + .gfx90a => .cdna2, + .gfx942 => .cdna3, + .gfx950 => .cdna4, + .gfx1030 => .rdna2, + .gfx1100, .gfx1101, .gfx1102, .gfx1103 => .rdna3, + .gfx1150 => .rdna3_5, + .gfx1200, .gfx1201 => .rdna4, + }; + } + }; + + /// Assumes homogeneous devices. + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + stdx.debug.assert(platform.target == .rocm, "computeCapability expects .rocm platform, got {}", .{platform.target}); + const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; const description = device.getDescription(platform.pjrt_api); const attributes = description.attributes(platform.pjrt_api); return for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), "compute_capability")) { - break attr.value().string; + break std.meta.stringToEnum(ComputeCapability, std.mem.sliceTo(attr.value().string, ':')); } } else null; } @@ -533,12 +1015,15 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = call_frame.args.buffers()[0]; + const buffer = buffers[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -550,7 +1035,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { } else return error.MemoryNotFound; var pjrt_buffer = try pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ - .data = buffer.data, + .device_buffer_ptr = buffer.data, .dims = shape.dims(), .element_type = pjrtx.bufferTypeFromDtype(shape.dtype()), .device = device, @@ -562,16 +1047,51 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); + defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); const name = call_frame.attrs.getByName(.string, "name").?.slice(); - log.info("{s}: {d}", .{ name, slice }); + std.debug.print("{s} [device={d}]: {d}\n", .{ name, device_ordinal, slice }); return null; } + +test "platform defaultMemoryLayout is boring" { + const platform = zml.testing.env(); + + const shapes = [_][]const i64{ + &.{4096}, + &.{ 4096, 4096 }, + &.{ 4096, 4096, 4096 }, + }; + for (shapes) |dims| { + // Checks that the PJRT client always return the same thing than `platform.defaultMemoryLayout` + // This allows to bypass the PJRT calls and the string of pjrt_client.defaultMemoryLayout. + const default_layout = try platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, pjrtx.bufferTypeFromDtype(.f32), dims); + const mem_layout = default_layout.toMemoryLayout(); + + if (platform.target == .furiosa) { + try std.testing.expectEqualDeep(mem_layout, platform.defaultMemoryLayout(dims, .f32)); + continue; + } + + // Note: I'm not just calling platform.defaultMemoryLayout because I'm investigating + // wether TPU requires its special branch. + try std.testing.expectEqualDeep(mem_layout, pjrt.MemoryLayout{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }); + } +} diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig new file mode 100644 index 00000000..d46a0c4c --- /dev/null +++ b/zml/profiling/profiler.zig @@ -0,0 +1,287 @@ +const std = @import("std"); +const log = std.log; + +const c = @import("c"); +const pjrt = @import("pjrt"); +const upb = @import("upb"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); +const zffi = @import("ffi"); + +pub fn profiler(api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const profile_options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); + try options.writeProto(upb_arena, profile_options); + + const serialized_options = try upb.serialize(profile_options, upb_arena); + const inner = try api.profiler(serialized_options); + + return try .init(allocator, io, inner, options); +} + +pub const ProfilerOptions = struct { + pub const default_version: u32 = 1; + + pub const DeviceType = enum(i32) { + unspecified = c.tensorflow_ProfileOptions_UNSPECIFIED, + cpu = c.tensorflow_ProfileOptions_CPU, + gpu = c.tensorflow_ProfileOptions_GPU, + tpu = c.tensorflow_ProfileOptions_TPU, + pluggable_device = c.tensorflow_ProfileOptions_PLUGGABLE_DEVICE, + }; + + pub const TraceOptions = struct { + host_traceme_filter_mask: ?u64 = null, + + fn isEmpty(self: TraceOptions) bool { + return self.host_traceme_filter_mask == null; + } + + fn writeProto(self: TraceOptions, proto: *c.tensorflow_ProfileOptions_TraceOptions) void { + if (self.host_traceme_filter_mask) |value| { + c.tensorflow_ProfileOptions_TraceOptions_set_host_traceme_filter_mask(proto, value); + } + } + }; + + pub const AdvancedConfigValue = union(enum) { + string: []const u8, + boolean: bool, + int64: i64, + + fn writeProto(self: AdvancedConfigValue, proto: *c.tensorflow_ProfileOptions_AdvancedConfigValue) void { + switch (self) { + .string => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_string_value(proto, upb.stringView(value)), + .boolean => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_bool_value(proto, value), + .int64 => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_int64_value(proto, value), + } + } + }; + + pub const AdvancedConfiguration = struct { + key: []const u8, + value: AdvancedConfigValue, + }; + + pub const defaults: ProfilerOptions = .{}; + + version: u32 = default_version, + repository_path: []const u8 = "/tmp/xprof", + session_id: []const u8 = "profiling", + device_type: ?DeviceType = .unspecified, + include_dataset_ops: ?bool = true, + host_tracer_level: ?u32 = 3, + device_tracer_level: ?u32 = 3, + python_tracer_level: ?u32 = 1, + enable_hlo_proto: ?bool = true, + start_timestamp_ns: ?u64 = null, + duration_ms: ?u64 = null, + trace_options: ?TraceOptions = null, + advanced_configuration: []const AdvancedConfiguration = &.{}, + raise_error_on_start_failure: ?bool = null, + override_hostname: ?[]const u8 = null, + + pub fn writeProto(self: ProfilerOptions, upb_arena: *c.upb_Arena, proto: *c.tensorflow_ProfileOptions) !void { + c.tensorflow_ProfileOptions_set_version(proto, self.version); + c.tensorflow_ProfileOptions_set_session_id(proto, upb.stringView(self.session_id)); + c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); + + if (self.device_type) |value| { + c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); + } + if (self.include_dataset_ops) |value| { + c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); + } + if (self.host_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_host_tracer_level(proto, value); + } + if (self.device_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_device_tracer_level(proto, value); + } + if (self.python_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_python_tracer_level(proto, value); + } + if (self.enable_hlo_proto) |value| { + c.tensorflow_ProfileOptions_set_enable_hlo_proto(proto, value); + } + if (self.start_timestamp_ns) |value| { + c.tensorflow_ProfileOptions_set_start_timestamp_ns(proto, value); + } + if (self.duration_ms) |value| { + c.tensorflow_ProfileOptions_set_duration_ms(proto, value); + } + if (self.trace_options) |trace_options| { + if (!trace_options.isEmpty()) { + const trace_proto = try upb.new(c.tensorflow_ProfileOptions_TraceOptions, upb_arena); + trace_options.writeProto(trace_proto); + c.tensorflow_ProfileOptions_set_trace_options(proto, trace_proto); + } + } + for (self.advanced_configuration) |entry| { + const value_proto = try upb.new(c.tensorflow_ProfileOptions_AdvancedConfigValue, upb_arena); + entry.value.writeProto(value_proto); + if (!c.tensorflow_ProfileOptions_advanced_configuration_set(proto, upb.stringView(entry.key), value_proto, upb_arena)) { + return error.OutOfMemory; + } + } + if (self.raise_error_on_start_failure) |value| { + c.tensorflow_ProfileOptions_set_raise_error_on_start_failure(proto, value); + } + if (self.override_hostname) |value| { + c.tensorflow_ProfileOptions_set_override_hostname(proto, upb.stringView(value)); + } + } +}; + +pub const Profiler = struct { + arena: std.heap.ArenaAllocator, + allocator: std.mem.Allocator, + io: std.Io, + inner: ?pjrt.Profiler, + start_timestamp_ns: ?u64, + stop_timestamp_ns: ?u64, + raise_error_on_start_failure: bool, + local_host_tracer_level: ?u32, + local_host_traceme_filter_mask: ?u64, + session_dir: []const u8, + profile: Profile, + + pub const Profile = struct { + protobuf_path: []const u8, + perfetto_path: []const u8, + }; + + pub fn init(allocator: std.mem.Allocator, io: std.Io, inner: ?pjrt.Profiler, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const session_dir = try std.Io.Dir.path.join(allocator, &.{ options.repository_path, "plugins", "profile", options.session_id }); + errdefer allocator.free(session_dir); + + try std.Io.Dir.cwd().createDirPath(io, session_dir); + + const protobuf_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.xplane.pb" }); + errdefer allocator.free(protobuf_path); + + const perfetto_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.trace.json" }); + errdefer allocator.free(perfetto_path); + + return .{ + .arena = arena, + .allocator = allocator, + .io = io, + .inner = inner, + .start_timestamp_ns = null, + .stop_timestamp_ns = null, + .raise_error_on_start_failure = options.raise_error_on_start_failure orelse false, + .local_host_tracer_level = options.host_tracer_level, + .local_host_traceme_filter_mask = if (options.trace_options) |trace_options| trace_options.host_traceme_filter_mask else null, + .session_dir = session_dir, + .profile = .{ + .protobuf_path = protobuf_path, + .perfetto_path = perfetto_path, + }, + }; + } + + pub fn deinit(self: *Profiler) void { + self.allocator.free(self.profile.protobuf_path); + self.allocator.free(self.profile.perfetto_path); + self.allocator.free(self.session_dir); + self.arena.deinit(); + } + + pub fn start(self: *Profiler) !void { + self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); + self.stop_timestamp_ns = null; + + if (shouldSkipPjrtProfiler()) { + log.warn("Detected SKIP_PJRT_PROFILER=true; disabling nested backend profiling and keeping local host tracing only.", .{}); + self.inner = null; + } else if (self.inner) |*inner| { + inner.start() catch |err| { + if (self.raise_error_on_start_failure) { + return err; + } + log.warn("Profiler backend failed to start; continuing with local host tracing only: {}", .{err}); + self.inner = null; + }; + } + if (self.local_host_tracer_level) |level| { + c.zml_traceme_session_start( + @intCast(level), + self.local_host_traceme_filter_mask orelse 0, + self.local_host_traceme_filter_mask != null, + ); + } + } + + pub fn stop(self: *Profiler) !?Profile { + self.stop_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); + + const protobuf = if (self.inner) |saved_inner| blk: { + var inner = saved_inner; + self.inner = null; + try inner.stop(); + break :blk try inner.collectData(self.arena.allocator()); + } else &.{}; + const merged_protobuf = c.zml_traceme_session_merge(zffi.ZigSlice.from(protobuf)); + defer if (merged_protobuf.len != 0) { + c.zml_traceme_str_free(merged_protobuf); + }; + const final_protobuf = if (merged_protobuf.len != 0) zffi.ZigSlice.to(u8, merged_protobuf) else protobuf; + if (final_protobuf.len == 0) { + return null; + } + + const normalized_protobuf = c.zml_traceme_session_postprocess( + zffi.ZigSlice.from(final_protobuf), + self.start_timestamp_ns orelse self.stop_timestamp_ns.?, + self.stop_timestamp_ns.?, + ); + defer if (normalized_protobuf.len != 0) { + c.zml_traceme_str_free(normalized_protobuf); + }; + if (normalized_protobuf.len == 0) { + log.err("Failed to normalize profile protobuf", .{}); + return error.ProfileTraceNormalizationFailed; + } + + const final_normalized_protobuf = zffi.ZigSlice.to(u8, normalized_protobuf); + + try self.writeFile(self.profile.protobuf_path, final_normalized_protobuf); + + xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile( + self.allocator, + self.io, + final_normalized_protobuf, + self.profile.perfetto_path, + ) catch |err| { + log.err("Failed to convert profile protobuf to Perfetto trace: {}", .{err}); + return error.ProfileTraceConversionFailed; + }; + + log.info("Profile dumped: {s} and {s}", .{ + self.profile.protobuf_path, + self.profile.perfetto_path, + }); + return self.profile; + } + + fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { + const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); + defer file.close(self.io); + + try file.writePositionalAll(self.io, contents, 0); + } +}; + +fn shouldSkipPjrtProfiler() bool { + const value = std.c.getenv("SKIP_PJRT_PROFILER") orelse return false; + return std.mem.eql(u8, std.mem.span(value), "true"); +} diff --git a/zml/profiling/traceme.h b/zml/profiling/traceme.h new file mode 100644 index 00000000..a1df3d04 --- /dev/null +++ b/zml/profiling/traceme.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct zml_traceme zml_traceme; + +zml_traceme *zml_traceme_start(zig_slice name); +void zml_traceme_stop(zml_traceme *traceme); +bool zml_traceme_enabled(void); +void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter); +zig_slice zml_traceme_session_merge(zig_slice xspace); +zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns); +void zml_traceme_str_free(zig_slice text); + +#ifdef __cplusplus +} +#endif diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc new file mode 100644 index 00000000..ca37dee7 --- /dev/null +++ b/zml/profiling/traceme_lib.cc @@ -0,0 +1,252 @@ +#include "zml/profiling/traceme.h" + +#include +#include +#include + +#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" +#include "xla/tsl/profiler/backends/cpu/host_tracer_utils.h" +#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" +#include "xla/tsl/profiler/utils/time_utils.h" +#include "xla/tsl/profiler/utils/xplane_utils.h" +#include "tsl/profiler/lib/traceme.h" +#include "tsl/profiler/protobuf/xplane.pb.h" + +#if defined(__APPLE__) +#include +#include +#endif + +#if defined(ZML_RUNTIME_CUDA) +#include "nvtx3/nvToolsExt.h" +#endif + +#if defined(ZML_RUNTIME_ROCM) +#include "rocprofiler-sdk-roctx/roctx.h" +#endif + +namespace { + +struct LocalHostTraceSession { + bool active = false; + uint64_t start_timestamp_ns = 0; +}; + +LocalHostTraceSession g_local_host_trace_session; + +tensorflow::profiler::XPlane* FindOrAddHostPlane( + tensorflow::profiler::XSpace* space) { + for (auto& plane : *space->mutable_planes()) { + if (plane.name() == "/host:CPU") { + return &plane; + } + } + tensorflow::profiler::XPlane* plane = space->add_planes(); + plane->set_name("/host:CPU"); + return plane; +} + +zig_slice CopyToOwnedSlice(const std::string& data) { + if (data.empty()) { + return {}; + } + void* ptr = std::malloc(data.size()); + if (ptr == nullptr) { + return {}; + } + std::memcpy(ptr, data.data(), data.size()); + return {ptr, data.size()}; +} + +#if defined(ZML_RUNTIME_CUDA) +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string& name) + : domain(TslNvtxDomain()), active(domain != nullptr) { + if (!active) { + return; + } + + nvtxEventAttributes_t attrs{}; + attrs.version = NVTX_VERSION; + attrs.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + attrs.messageType = NVTX_MESSAGE_TYPE_ASCII; + attrs.message.ascii = name.c_str(); + nvtxDomainRangePushEx(domain, &attrs); + } + + ~CudaNvtxRangeBackend() { + if (active) { + nvtxDomainRangePop(domain); + } + } + + static bool IsEnabled() { return TslNvtxDomain() != nullptr; } + + private: + static nvtxDomainHandle_t TslNvtxDomain() { + static nvtxDomainHandle_t domain = nvtxDomainCreateA("TSL"); + return domain; + } + + nvtxDomainHandle_t domain = nullptr; + bool active = false; +}; +#else +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(ZML_RUNTIME_ROCM) +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string& name) + : active(RangePush() != nullptr && RangePop() != nullptr) { + if (!active) { + return; + } + + const int level = RangePush()(name.c_str()); + active = level >= 0; + } + + ~RocmRoctxRangeBackend() { + if (active) { + RangePop()(); + } + } + + static bool IsEnabled() { + return RangePush() != nullptr && RangePop() != nullptr; + } + + private: + using RangePushFn = decltype(&roctxRangePushA); + using RangePopFn = decltype(&roctxRangePop); + + static RangePushFn RangePush() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePushA")); + } + + static RangePopFn RangePop() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePop")); + } + + bool active = false; +}; +#else +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(__APPLE__) +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string& name) + : log(Logger()), signpost_id(os_signpost_id_generate(log)) { + os_signpost_interval_begin(log, signpost_id, "zml.scope", "%{public}s", + name.c_str()); + } + + ~MacOsSignpostRangeBackend() { + os_signpost_interval_end(log, signpost_id, "zml.scope", "%{public}s", + ""); + } + + static bool IsEnabled() { return true; } + + private: + static os_log_t Logger() { + static os_log_t log = + os_log_create("ai.zml", OS_LOG_CATEGORY_POINTS_OF_INTEREST); + return log; + } + + os_log_t log; + os_signpost_id_t signpost_id; +}; +#else +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +} // namespace + +struct zml_traceme { + explicit zml_traceme(std::string encoded_name) + : name(std::move(encoded_name)), + traceme([&] { return name; }), + cuda_backend(name), + rocm_backend(name), + macos_backend(name) {} + + std::string name; + tsl::profiler::TraceMe traceme; + CudaNvtxRangeBackend cuda_backend; + RocmRoctxRangeBackend rocm_backend; + MacOsSignpostRangeBackend macos_backend; +}; + +extern "C" zml_traceme* zml_traceme_start(zig_slice name) { + return new zml_traceme( + std::string(static_cast(name.ptr), name.len)); +} + +extern "C" void zml_traceme_stop(zml_traceme* traceme) { delete traceme; } + +extern "C" bool zml_traceme_enabled(void) { + return tsl::profiler::TraceMe::Active() || + CudaNvtxRangeBackend::IsEnabled() || + RocmRoctxRangeBackend::IsEnabled() || + MacOsSignpostRangeBackend::IsEnabled(); +} + +extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter) { + g_local_host_trace_session.start_timestamp_ns = + tsl::profiler::GetCurrentTimeNanos(); + g_local_host_trace_session.active = + enable_filter ? tsl::profiler::TraceMeRecorder::Start(level, filter_mask) + : tsl::profiler::TraceMeRecorder::Start(level); +} + +extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return CopyToOwnedSlice( + std::string(static_cast(xspace.ptr), xspace.len)); + } + + if (g_local_host_trace_session.active) { + auto events = tsl::profiler::TraceMeRecorder::Stop(); + g_local_host_trace_session.active = false; + if (!events.empty()) { + tsl::profiler::ConvertCompleteEventsToXPlane( + g_local_host_trace_session.start_timestamp_ns, std::move(events), + FindOrAddHostPlane(&space)); + } + } + + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return {}; + } + + tsl::profiler::SetXSpacePidIfNotSet(space, getpid()); + tsl::profiler::PostProcessSingleHostXSpace(&space, start_time_ns, + stop_time_ns); + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" void zml_traceme_str_free(zig_slice text) { std::free(text.ptr); } diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig new file mode 100644 index 00000000..5aed01df --- /dev/null +++ b/zml/profiling/tracer.zig @@ -0,0 +1,235 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const c = @import("c"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); +const zffi = @import("ffi"); + +pub const Span = struct { + inner: ?*c.zml_traceme = null, + + /// Name is copied underneath + pub fn start(name: []const u8) Span { + return .{ + .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), + }; + } + + pub fn end(self: *Span) void { + if (self.inner) |inner| { + c.zml_traceme_stop(inner); + self.inner = null; + } + } +}; + +// TraceMe is always the base host tracing path. The C tracing bridge adds NVTX +// ranges on CUDA/Linux builds, roctx ranges on ROCm/Linux builds, and +// os_signpost intervals on macOS. +pub const supportsDeviceAnnotations = switch (builtin.os.tag) { + .macos => true, + .linux => platforms.target == .cuda or platforms.target == .rocm, + else => false, +}; + +pub fn enabled() bool { + return c.zml_traceme_enabled(); +} + +/// Creates a Span with encoded metadata. The metadata encoded size must be known at compile time. +/// If it's not your case, use `formatSpanName` to generate the span name before and use void for the metadata. +pub fn span(comptime name: []const u8, metadata: anytype) Span { + switch (@typeInfo(@TypeOf(metadata))) { + .void => { + return .start(name); + }, + .@"struct" => |info| { + if (info.field_names.len == 0) { + return .start(name); + } + + var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; + var encoded: std.ArrayList(u8) = .initBuffer(&buffer); + + appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; + return .start(encoded.items); + }, + else => @compileError("Unsupported metadata."), + } +} + +/// Returns an allocator-owned encoded span name for metadata that cannot be +/// bounded by `span` at comptime. The caller owns the returned slice. +pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { + switch (@typeInfo(@TypeOf(metadata))) { + .@"struct" => { + var encoded: std.ArrayList(u8) = .empty; + errdefer encoded.deinit(allocator); + + try appendEncodedName(&encoded, allocator, name, metadata); + return try encoded.toOwnedSlice(allocator); + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { + try buffer.appendSlice(allocator, name); + + var field_count: usize = 0; + try buffer.append(allocator, '#'); + inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { + if (shouldEncodeField(field_name, @field(metadata, field_name))) { + if (field_count != 0) { + try buffer.append(allocator, ','); + } + field_count += 1; + + const key = comptime metadataKey(field_name); + validateMetadataToken(key); + try buffer.appendSlice(allocator, key); + try buffer.append(allocator, '='); + try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); + } + } + + // We may not have written any fields because of optional values. + if (field_count == 0) { + buffer.items.len = name.len; + } else { + try buffer.append(allocator, '#'); + } +} + +fn validateMetadataToken(token: []const u8) void { + if (std.mem.indexOfScalar(u8, token, '#') != null) { + @panic("trace metadata cannot contain '#'"); + } +} + +fn metadataKey(comptime field_name: []const u8) []const u8 { + return if (comptime std.mem.eql(u8, field_name, "root")) "_r" else field_name; +} + +fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { + switch (@typeInfo(Metadata)) { + .@"struct" => |info| { + if (info.field_names.len == 0) return name.len; + + // name#[attrs]# + var max_len = name.len + 2; + inline for ( + info.field_names, + info.field_types, + info.field_attrs, + ) |field_name, FieldType, field_attr| { + const key = comptime metadataKey(field_name); + validateMetadataToken(key); + + const default: ?FieldType = field_attr.defaultValue(FieldType); + // [key]=[value], + max_len += key.len + 2; + max_len += switch (@typeInfo(FieldType)) { + .bool => 1, + .int => @max( + std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, + ), + .@"enum" => |ty| b: { + var max_enum_len: usize = 0; + for (ty.field_names) |enum_field_name| { + max_enum_len = @max(max_enum_len, enum_field_name.len); + } + break :b max_enum_len; + }, + .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, + .enum_literal => @tagName(default.?).len, + else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), + }; + } + return max_len; + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { + const FieldType = @TypeOf(value); + return switch (@typeInfo(FieldType)) { + .optional => value != null, + .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, + else => true, + }; +} + +fn appendMetadataValue(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, value: anytype, comptime field_name: []const u8) !void { + const T = @TypeOf(value); + switch (@typeInfo(T)) { + .optional => { + try appendMetadataValue(buffer, allocator, value.?, field_name); + }, + .bool => { + if (comptime std.mem.eql(u8, field_name, "root")) { + try buffer.append(allocator, '1'); + } else { + try buffer.appendSlice(allocator, if (value) "1" else "0"); + } + }, + .int, .comptime_int => try buffer.print(allocator, "{}", .{value}), + .float, .comptime_float => try buffer.print(allocator, "{d}", .{value}), + .enum_literal => try buffer.appendSlice(allocator, @tagName(value)), + .@"enum" => try buffer.appendSlice(allocator, @tagName(value)), + .pointer => |ptr| { + if (ptr.size == .slice and ptr.child == u8) { + validateMetadataToken(value); + try buffer.appendSlice(allocator, value); + return; + } + + if (ptr.size == .one) switch (@typeInfo(ptr.child)) { + .array => |arr| { + if (arr.child == u8) { + const slice = value[0..]; + validateMetadataToken(slice); + try buffer.appendSlice(allocator, slice); + return; + } + }, + else => {}, + }; + + @compileError("trace metadata pointers must be []const u8 or string literals"); + }, + .array => |arr| { + if (arr.child != u8) { + @compileError("trace metadata arrays must be [N]u8"); + } + validateMetadataToken(value[0..]); + try buffer.appendSlice(allocator, value[0..]); + }, + else => @compileError("unsupported trace metadata type for field '" ++ field_name ++ "'"), + } +} + +test "formatspanName encodes metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = true, + .count = @as(u8, 42), + .label = "batch", + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span#_r=1,count=42,label=batch#", formatted); +} + +test "formatspanName omits empty metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = false, + .optional = @as(?u8, null), + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span", formatted); +} diff --git a/zml/quantization.zig b/zml/quantization.zig new file mode 100644 index 00000000..db20619a --- /dev/null +++ b/zml/quantization.zig @@ -0,0 +1,519 @@ +//! Quantization metadata and scheme classification. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("zml.zig"); + +const DataType = @import("dtype.zig").DataType; +const platform_mod = @import("platform.zig"); +const Platform = platform_mod.Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +pub const nvfp4_block_size = 16; +pub const mx_block_size = 32; + +pub const Quantization = struct { + scheme: Scheme, // decided when the checkpoint is read + scales: Tensor, + global_scale: ?GlobalScale = null, + input_scale: ?GlobalScale = null, + /// HACK: `scales` is stored in the 128x4 blocked layout of Blackwell's block-scaled mma. + // Something cleaner will need to be done to differentiate different kinds of swizzling + swizzled_scales: bool = false, + + /// A per-tensor scale and the polarity its producer wrote it in + pub const GlobalScale = struct { + value: Tensor, + operation: Operation, + + pub const Operation = enum { multiply, divide }; + + pub fn asMultiplier(self: GlobalScale) Tensor { + const s = self.value.convert(.f32); + const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization + + return switch (self.operation) { + .multiply => flat, + .divide => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), + }; + } + }; + + pub const Scheme = enum { + /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. + /// Emitted by llm-compressor and by NVIDIA's ModelOpt. + nvfp4, + /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values + mxfp8, + /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. + // GPT-OSS like + mxfp4, + /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. + /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. + fp8_per_channel, + /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. + /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. + fp8_block128, + /// E4M3FN or E4M3FNUZ weights with E8M0, BF16 or F32 scales per 32x32 tile. + fp8_block32, + /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a + /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. + fp8_per_tensor, + + pub fn accepts(self: Scheme, weight: Shape, scale: Shape) bool { + if (weight.rank() != 2) return false; + + const n = weight.dim(0); + const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); + + return switch (self) { + .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + scale.dtype() == .f8e4m3fn and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, + .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and + isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, + .fp8_per_channel => weight.dtype() == .f8e4m3fn and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) == 1, + .fp8_block32 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz) and + (scale.dtype() == .f8e8m0 or ((scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1)) and + scale.rank() == 2 and scale.dim(0) * 32 == n and scale.dim(1) * 32 == k, + .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == (std.math.divCeil(i64, n, 128) catch unreachable) and + scale.dim(1) == (std.math.divCeil(i64, k, 128) catch unreachable), + }; + } + + pub fn classify(weight: Shape, scale: Shape) ?Scheme { + for (std.enums.values(Scheme)) |scheme| { + if (scheme.accepts(weight, scale)) return scheme; + } + + return null; + } + + pub fn isMx(self: Scheme) bool { + return self == .mxfp8 or self == .mxfp4; + } + }; +}; + +pub const QuantizedInput = struct { + values: Tensor, + scales: Tensor, + global_scale: ?Tensor, + + pub fn reuseBuffer(self: QuantizedInput, other: QuantizedInput) QuantizedInput { + return .{ + .values = self.values.reuseBuffer(other.values), + .scales = self.scales.reuseBuffer(other.scales), + .global_scale = if (self.global_scale != null and other.global_scale != null) self.global_scale.?.reuseBuffer(other.global_scale.?) else self.global_scale, + }; + } +}; + +pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { + const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; + + return switch (quantization.scheme) { + .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, + .fp8_block32, .fp8_block128 => blk: { + const dtype: DataType = switch (platform.target) { + .cuda => .f8e4m3fn, + // RNGD contracts matching FP8 pairs in 128-wide K blocks. + .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, + } else return null, + else => return null, + }; + + break :blk switch (quantization.scheme) { + .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), + .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), + else => unreachable, + }; + }, + .mxfp8, .mxfp4, .fp8_per_channel, .fp8_per_tensor => null, + }; +} + +pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { + stdx.debug.assert(@mod(x.dim(axis), mx_block_size) == 0, "MXFP4 activation width must be divisible by {}, got {f}", .{ mx_block_size, x.shape() }); + + const grouped = x.convert(.f32).splitAxis(axis, .{ .mx_ks = -1, .mx_block = mx_block_size }); + const raw_scale = grouped.abs().max(.mx_block).maximum(.scalar(6 * 0x1p-126, .f32)).scale(1.0 / 6.0); + const scales = ceilPowerOfTwo(raw_scale); + + return .{ + .values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-6, .f32), .scalar(6, .f32)) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)), + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), mx_block_size)).withDtype(.f32)).convert(.f8e8m0), + .global_scale = null, + }; +} + +pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { + const platform = zml.Compiler.current().platform; + + const dtype: ?DataType = switch (platform.target) { + .cuda, .cpu, .tpu => .f8e4m3fn, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null, + else => null, + }; + + if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); + return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); +} + +fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { + const bits = raw_scale.bitCast(.u32); + return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); +} + +/// Quantize activation blocks, preserving the input's axes. +/// E8M0 scales round upward to a power of two to avoid overflowing FP8 values. +pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataType, scale_dtype: DataType) QuantizedInput { + stdx.debug.assert(block_size > 0, "block FP8 size must be positive, got {}", .{block_size}); + stdx.debug.assert(@mod(x.dim(axis), block_size) == 0, "block FP8 activation width must be divisible by {}, got {f}", .{ block_size, x.shape() }); + // Match the routed-MoE quantizer's FNUZ range. + const fp8_max: f32 = switch (dtype) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => stdx.debug.panic("expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}), + }; + const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = block_size }); + const scales = switch (scale_dtype) { + // Put scale arithmetic before the reduction so it can fuse into the producer. + .f32 => grouped.abs() + .maximum(.scalar(1e-10, .f32)) + .scale(1.0 / fp8_max) + .max(.fp8_block), + .f8e8m0 => blk: { + const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); + break :blk ceilPowerOfTwo(raw_scale); + }, + else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), + }; + const values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-fp8_max, .f32), .scalar(fp8_max, .f32)) + .convert(dtype) + .reshape(x.shape().withDtype(dtype)); + return .{ + .values = values, + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), block_size)).withDtype(.f32)).convert(scale_dtype), + .global_scale = null, + }; +} + +pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedInput { + const value_max = 6.0; + const scale_min_normal = 0x1p-6; + const scale_max = DataType.f8e4m3fn.maxValue().as(f32); + + stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); + stdx.debug.assert(@rem(x.dim(axis), nvfp4_block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, nvfp4_block_size, x.shape() }); + + const dt = x.dtype(); + const scaled = if (input_global_scale) |igs| + x.div(igs.convert(dt).broad(x.shape())) + else + x; + const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = nvfp4_block_size }); + + const amax = grouped.abs() + .scale(1.0 / value_max) + .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) + .max(.blk); + + const scales = amax.convert(.f8e4m3fn); + + const divisor = scales.convert(dt) + .maximum(.scalar(scale_min_normal, dt)) + .broad(grouped.shape()); + + const values = grouped.div(divisor) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)); + + return .{ + .values = values, + .scales = scales.squeeze(.blk), + .global_scale = input_global_scale, + }; +} + +fn supportsNvfp4InputQuantization(platform: *const Platform) bool { + return if (platform_mod.cuda.computeCapability(platform)) |cc| cc.atLeast(.{ .major = 10, .minor = 0 }) else false; +} + +fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); +} + +fn isMxScale(scale: Shape) bool { + return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); +} + +// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a +// scaled-dot are still open; see the scaledDot doc comment). +test "Quantization.Scheme.classify" { + const expect = std.testing.expectEqual; + + // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under + // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. + const nvfp4_packed: Shape = .init(.{ .dout = 17408, .d = 2560 }, .u8); + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 17408, .sc = 1 }, .bf16), + )); + + // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 21504, .d = 2688 }, .u8), + .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), + )); + + // Native (unpacked) f4e2m1, with K no longer doubled. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), + )); + + // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 136, .sc = 40 }, .bf16), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 40, .sc = 48 }, .bf16), + )); + + inline for (.{ .f8e4m3fn, .f8e4m3fnuz }) |dtype| { + try expect(@as(?Quantization.Scheme, .fp8_block32), Quantization.Scheme.classify( + .init(.{ .dout = 1280, .d = 5120 }, dtype), + .init(.{ .dout = 40, .sc = 160 }, .f8e8m0), + )); + } + + // GLM uses f32 scales and permits a partial final 128-row tile. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 16, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e8m0), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + + // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); + + // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half + // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives + // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .u8), + )); + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), + .init(.{ .dout = 202048, .sc = 208 }, .u8), + )); + // The same tensor from a store that types the scale natively. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), + )); + // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 1248 }, .u8), + )); + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 312 }, .u8), + )); + + // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with + // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); + // DeepSeek V4 stores the same packed FP4 bits in signed bytes. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .d = 2048 }, .i8), + .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), + )); + // Native (unpacked) f4e2m1, K no longer halved. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + // An e8m0 scale never turns a non-fp4/fp8 weight into MX. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .bf16), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + + // Rejected: everything no backend here can express. + const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); + // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); + // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); + // Per-channel in f32: what fusing per-tensor projections produces. The Metal + // per-channel kernels have f32 entries, so this is a real scheme, not a decline. + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); + // A block grid that is neither per-channel nor 128x128. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); + // Stacked MoE weights are rank 3 and go through the MoE path, not this one. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .expert = 128, .dout = 1408, .d = 1408 }, .u8), + .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), + )); + + // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] + // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and + // `mxfp8` is declared first + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .u8), + )); + + // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the + // `count() > 1` guard on the grids is what makes per-tensor win. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify( + .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .bf16), + )); +} + +test "block FP8 quantization preserves axes and reconstructs constant blocks in FN and FNUZ" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, 128, dtype, .f32); + return .{ .values = input.values.convert(.f32), .scales = input.scales }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .bf16); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 2).withDtype(.f32), exe.output_shapes[1]); + var host: [2 * 256 * 3]zml.floats.BFloat16 = undefined; + for (&host, 0..) |*value, i| { + const magnitude: f32 = if ((i / 3) % 256 < 128) 1.0 else 2.0; + value.* = .fromF32(switch (i % 3) { + 0 => 0.0, + 1 => magnitude, + else => -magnitude, + }); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); + defer buffer.deinit(); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + for (scales.constItems(f32)) |scale| try std.testing.expect(std.math.isFinite(scale) and scale > 0); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 6 + ((i / 3) % 256) / 128 * 3 + i % 3; + try std.testing.expectApproxEqAbs(host[i].toF32(), value * scales.constItems(f32)[scale_index], 1e-6); + } + } +} + +test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, 32, dtype, .f8e8m0); + return .{ .values = input.values.convert(.f32), .scales = input.scales.convert(.f32) }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); + const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; + var host: [2 * 256 * 3]f32 = undefined; + var maxima: [2 * 8 * 3]f32 = @splat(0); + for (&host, 0..) |*value, i| { + const block = (i / 3) % 256 / 32; + const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; + const boundary = fp8_max * std.math.pow(f32, 2, @as(f32, @floatFromInt(block)) - 4); + const bits: u32 = @bitCast(boundary); + const magnitude: f32 = switch (i % 3) { + 0 => @bitCast(bits - 1), + 1 => boundary, + else => @bitCast(bits + 1), + }; + value.* = if (block == 0) 0 else if (block == 1) 1e-30 else magnitude * (if (i % 2 == 0) @as(f32, 1) else -1); + maxima[scale_index] = @max(maxima[scale_index], @abs(value.*)); + } + var expected_scales: [2 * 8 * 3]f32 = undefined; + for (&expected_scales, maxima) |*scale, maximum| { + const raw = @max(maximum, 1e-4) * (1.0 / fp8_max); + const bits: u32 = @bitCast(raw); + const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); + scale.* = @bitCast(exponent << 23); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); + defer buffer.deinit(); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + try std.testing.expectEqualSlices(f32, &expected_scales, scales.constItems(f32)); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 24 + ((i / 3) % 256) / 32 * 3 + i % 3; + const normalized = std.math.clamp(host[i] / expected_scales[scale_index], -fp8_max, fp8_max); + const expected = if (dtype == .f8e4m3fn) zml.floats.Float8E4M3FN.fromF32(normalized).toF32() else zml.floats.Float8E4M3FNUZ.fromF32(normalized).toF32(); + try std.testing.expectEqual(expected, value); + } + } +} diff --git a/zml/safetensors.zig b/zml/safetensors.zig index bb14c334..3827fe27 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,67 +2,337 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; -const Dims = Shape.DimsArray; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); -pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -const BYTES_HEADER = 8; +// TODO: consider promoting this to the top level struct +// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights +/// Maps tensor names to their shape and positions in different sub files. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, -pub const FileType = enum { - index, - safetensors, - unknown, + tensors: std.StringArrayHashMapUnmanaged(Tensor), + + /// Metadata found in `__metadata__` field of the safetensors header. + /// This is rarely use and the information there is not standardized, so we let user code handle it. + /// Here we just store the plain bytes, reach out if you need this to be already parsed. + metadata: []const u8, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = "", + }; + } + + pub fn deinit(self: *TensorRegistry) void { + self.arena.deinit(); + } + + pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { + const file_type, const file = try resolveModelEntrypoint(io, repo); + defer file.close(io); + + var registry: TensorRegistry = .init(allocator); + switch (file_type) { + .safetensors => try registry.parseSafetensors(io, allocator, file), + .index => try registry.parseIndexJson(io, allocator, repo, file), + .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), + .unknown => return error.InvalidPath, + } + return registry; + } + + pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { + const file_type: FileType = .resolve(path); + switch (file_type) { + .safetensors => { + // open directly + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + var registry: TensorRegistry = .init(allocator); + try registry.parseSafetensors(io, allocator, file); + return registry; + }, + .full_index, + .index, + => { + // resolve repo as parent folder then open + const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + + var registry: TensorRegistry = .init(allocator); + try switch (file_type) { + .index => registry.parseIndexJson(io, allocator, repo, file), + .full_index => registry.parseFullIndexJson(io, allocator, repo, file), + .safetensors, .unknown => unreachable, + }; + return registry; + }, + .unknown => { + // the path is possibly a folder, let's try that + const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return try fromRepo(allocator, io, repo); + }, + } + } + + /// Register a tensor, duplicate tensor name and uri. + /// If you create a lot of tensors sharing the same uri, + /// considere calling `registry.tensors` directly and avoid duplicated uris. + pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { + const allocator = self.arena.allocator(); + + const owned_name = try allocator.dupe(u8, tensor.name); + errdefer allocator.free(owned_name); + const owned_uri = try allocator.dupe(u8, tensor.file_uri); + errdefer allocator.free(owned_uri); + + const owned_tensor: Tensor = .{ + .name = owned_name, + .file_uri = owned_uri, + .shape = tensor.shape, + .offset = tensor.offset, + }; + try self.tensors.put(allocator, owned_name, owned_tensor); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } + + /// Parse a given .safetensors file and create entries for each tensor + pub fn parseSafetensors( + self: *TensorRegistry, + io: std.Io, + scratch_allocator: std.mem.Allocator, + file: std.Io.File, + ) !void { + var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); + defer tmp_state.deinit(); + const tmp_arena = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); + const file_name_len = try file.realPath(io, owned_file_name); + // Try to release extra memory, but we don't care if it fails + _ = self_arena.resize(owned_file_name, file_name_len); + owned_file_name = owned_file_name[0..file_name_len]; + + const header_buf = try tmp_arena.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = file.reader(io, header_buf); + + const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); + const BYTES_HEADER = 8; + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); + + try file_reader.interface.readSliceAll(json_data); + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + tmp_arena, + json_data, + .{}, + ); + + try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); + for (headers.object.keys(), headers.object.values()) |key, value| { + if (std.mem.eql(u8, key, "__metadata__")) { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + continue; + } + + const owned_name = try self_arena.dupe(u8, key); + const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); + self.tensors.putAssumeCapacity(owned_name, tensor); + } + } + + pub fn parseIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; + try safetensors_files.ensureUnusedCapacity(tmp, 64); + if (index.object.get("weight_map")) |wm| { + for (wm.object.values()) |file_name| { + try safetensors_files.put(tmp, file_name.string, {}); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + for (safetensors_files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try self.parseSafetensors(io, tmp, file); + } + } + + pub fn parseFullIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + + var repo_uri_buf = try tmp.alloc(u8, 1024); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + + // Note: this is relatively inefficient, we read the full file, copy most of it's content, + // then reduce to TensorRegistry format. + // We could update TensorRegistry as we stream the json. + // Zig std json API are a bit awkward around streaming though. + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try realpaths.ensureUnusedCapacity(tmp, 64); + const wm = index.object.get("weight_map") orelse return error.InvalidJson; + + try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); + for (wm.object.keys(), wm.object.values()) |key, value| { + const tensor_name = try self_arena.dupe(u8, key); + const object = value.object; + const filename = value.object.get("filename").?.string; + const filename_entry = try realpaths.getOrPut(tmp, filename); + const file_uri = if (filename_entry.found_existing) + filename_entry.value_ptr.* + else uri: { + const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); + filename_entry.value_ptr.* = uri; + break :uri uri; + }; + + const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); + self.tensors.putAssumeCapacity(tensor_name, tensor_info); + } + + if (index.object.get("metadata")) |metadata| { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + } + } + + pub fn saveAsFullIndexJson( + self: *const TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + out: *std.Io.Writer, + ) !void { + var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); + defer gpa.free(repo_uri_buf); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + var total_size: u64 = 0; + for (self.tensors.values()) |value| total_size += value.byteSize(); + + try out.print( + \\{{ + \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, + \\"weight_map": {{ + \\ + , .{ self.tensors.count(), total_size }); + + const n = self.tensors.count(); + for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { + std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); + const local_name = value.file_uri[repo_name.len + 1 ..]; + const end_offset = value.offset + value.byteSize(); + const safetensors_dtype: []const u8 = dt: { + for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { + if (zml_dt == value.shape._dtype) break :dt st_dt; + } + // Note: we could bitcast to u8, + // but reaching this panic implies the user didn't loaded this TensorRegistry + // from a safetensors file, and manually created the dict. + // In which case, they know what they are doing, + // and can easily insert the bitcast themselves. + std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); + }; + const comma = if (i + 1 == n) "" else ","; + try out.print( + \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} + \\ + , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); + } + + try out.writeAll( + \\} + \\} + \\ + ); + try out.flush(); + } }; -pub fn fetchRegistry( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - model: std.Io.File, -) !TensorRegistry { - const file_type = try resolveFiletype(io, model); - - return switch (file_type) { - .index => blk: { - const file_reader_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(file_reader_buf); - - var index_reader = model.reader(io, file_reader_buf); - - var safetensors_index = try parseSafetensorsIndex( - allocator, - &index_reader.interface, - ); - defer safetensors_index.deinit(); - - var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); - - try parseSafetensorsIndexFiles( - allocator, - io, - ®istry, - safetensors_index, - repo, - ); - - break :blk registry; - }, - .safetensors => blk: { - var registry: TensorRegistry = .init(allocator); - - try parseSafetensors(allocator, io, ®istry, model); - - break :blk registry; - }, - else => return error.InvalidPath, - }; -} - pub const TensorReader = struct { tensor: Tensor, file: std.Io.File, @@ -74,13 +344,15 @@ pub const TensorReader = struct { io: std.Io, interface: std.Io.Reader, + pub const InitOpts = struct { + alignment: ?std.mem.Alignment = null, + }; + pub fn init( io: std.Io, tensor: Tensor, buffer: []u8, - opts: struct { - alignment: ?std.mem.Alignment = null, - }, + opts: InitOpts, ) !TensorReader { const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); errdefer file.close(io); @@ -126,13 +398,17 @@ pub const TensorReader = struct { self.file.close(self.io); } + // Limit single reads to 1GB to avoid issues on macOS where pread returns + // INT_MAX when the requested size is larger than 2GB + const max_single_read_bytes: u64 = 1 << 30; + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *TensorReader = @fieldParentPtr("interface", r); if (self.remaining == 0) { return error.EndOfStream; } - const combined_limit = limit.min(.limited64(self.remaining)); + const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); const n = try self.file_reader.interface.stream(w, combined_limit); self.remaining -= n; return n; @@ -156,7 +432,7 @@ pub const TensorReader = struct { return error.EndOfStream; } - const requested_bytes = limit.minInt64(self.remaining); + const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); if (self.padding_remaining > 0) { _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); @@ -227,6 +503,34 @@ pub const Tensor = struct { return self.shape.byteSize(); } + pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { + return TensorReader.init(io, self, buffer, opts); + } + + pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { + const shape_field = object.get("shape").?.array; + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(object.get("dtype").?.string); + + var dims: Shape.DimsArray = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + return .{ + .file_uri = file_uri, + .name = name, + .shape = .init(dims.slice(), dtype), + .offset = base_offset + start, + }; + } + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -238,621 +542,100 @@ pub const Tensor = struct { } }; -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +pub const FileType = enum { + full_index, + index, + safetensors, + unknown, - tensors: Tensors, - metadata: Metadatas, - - mutex: std.Io.Mutex = .init, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .{}, - }; - } - - pub fn initWithMetadata( - allocator: std.mem.Allocator, - metadata: Metadatas, - ) !TensorRegistry { - var self: TensorRegistry = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = .empty, - }; - - try self.mergeMetadata(metadata); - - return self; - } - - pub fn fromRepo( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - ) !TensorRegistry { - const entrypoint = try resolveModelEntrypoint(io, repo); - return try fetchRegistry(allocator, io, repo, entrypoint); - } - - pub fn fromPath( - allocator: std.mem.Allocator, - io: std.Io, - path: []const u8, - ) !TensorRegistry { - var repo = try resolveModelRepo(io, path); - - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - return try fetchRegistry(allocator, io, repo, try repo.openFile(io, path, .{ .mode = .read_only })); + pub fn resolve(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { + return .full_index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; } else { - const entrypoint = try resolveModelEntrypoint(io, repo); - return try fetchRegistry(allocator, io, repo, entrypoint); + return .unknown; } } - - pub fn deinit(self: *TensorRegistry) void { - const allocator = self.arena.allocator(); - self.tensors.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } - - pub fn mergeMetadata( - self: *TensorRegistry, - other: Metadatas, - ) !void { - // self.mutex.lock(); - // defer self.mutex.unlock(); - - const allocator = self.arena.allocator(); - - var it = other.iterator(); - while (it.next()) |entry| { - const key = try allocator.dupe(u8, entry.key_ptr.*); - const value = try entry.value_ptr.*.clone(allocator); - - const gop = try self.metadata.getOrPut(allocator, key); - if (gop.found_existing) { - gop.value_ptr.*.deinit(allocator); - gop.value_ptr.* = value; - log.info("Overwrote existing metadata key={s} with value={f}", .{ key, value }); - } else { - gop.value_ptr.* = value; - log.debug("Added new metadata key={s} with value={f}", .{ key, value }); - } - } - } - - pub fn registerTensor( - self: *TensorRegistry, - tensor: Tensor, - ) !void { - const allocator = self.arena.allocator(); - - // self.mutex.lock(); - // defer self.mutex.unlock(); - - var tensor_copy = tensor; - - tensor_copy.name = try allocator.dupe(u8, tensor.name); - tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); - - try self.tensors.put(allocator, tensor_copy.name, tensor_copy); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn iterator(self: *TensorRegistry) Tensors.Iterator { - return self.tensors.iterator(); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } }; -pub fn parseSafetensors( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - file: std.Io.File, -) !void { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const header_buf = try allocator.alloc(u8, 64 * 1024); - defer allocator.free(header_buf); - - var reader = file.reader(io, header_buf); - - var file_uri_buf: [1024]u8 = undefined; - const file_uri_len = try file.realPath(io, &file_uri_buf); - const file_uri = file_uri_buf[0..file_uri_len]; - - const json_header_length: u64 = try reader.interface.takeInt(u64, .little); - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); - - try reader.interface.readSliceAll(json_data); - - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - arena.allocator(), - json_data, - .{}, - ); - - var it = headers.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - const value = entry.value_ptr.*; - - if (std.mem.eql(u8, key, "__metadata__")) { - // Parse metadata into a local arena to avoid concurrent allocations - // against registry.arena (which is not thread-safe). - var local_arena = std.heap.ArenaAllocator.init(allocator); - defer local_arena.deinit(); - - const metas = try parseMetadata(local_arena.allocator(), value); - try registry.mergeMetadata(metas); - - continue; - } - - const shape_field = value.object.get("shape").?.array; - - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = value.object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(value.object.get("dtype").?.string); - - var dims: Dims = .{}; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - const tensor: Tensor = .{ - .file_uri = file_uri, - .name = key, - .shape = .init(dims.slice(), dtype), - .offset = data_start_offset + start, - }; - - try registry.registerTensor(tensor); - } -} - -const SafetensorsIndex = struct { - pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); - - arena: std.heap.ArenaAllocator, - - map: Map, - metadata: Metadatas, - - pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .map = .empty, - .metadata = .empty, - }; - } - - pub fn iterator(self: *SafetensorsIndex) Map.Iterator { - return self.map.iterator(); - } - - pub fn deinit(self: *SafetensorsIndex) void { - const allocator = self.arena.allocator(); - self.map.deinit(allocator); - self.metadata.deinit(allocator); - self.arena.deinit(); - } -}; - -fn parseSafetensorsIndex( - allocator: std.mem.Allocator, - reader: *std.Io.Reader, -) !SafetensorsIndex { - var safetensors_index: SafetensorsIndex = .init(allocator); - errdefer safetensors_index.deinit(); - - const arena_allocator = safetensors_index.arena.allocator(); - var json_reader: std.json.Reader = .init(arena_allocator, reader); - - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - arena_allocator, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - const weight_map = index.object.get("weight_map"); - - if (weight_map) |wm| { - var it = wm.object.iterator(); - - while (it.next()) |entry| { - const weight_name = entry.key_ptr.*; - const filename = entry.value_ptr.string; - - const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); - - if (!map_entry.found_existing) { - map_entry.value_ptr.* = .{}; - } - - try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - if (index.object.get("metadata")) |metadata_val| { - safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); - } - - return safetensors_index; -} - -fn parseSafetensorsIndexFiles( - allocator: std.mem.Allocator, - io: std.Io, - registry: *TensorRegistry, - safetensors_index: SafetensorsIndex, - repo: std.Io.Dir, -) !void { - const filenames = safetensors_index.map.keys(); - for (filenames) |filename| { - const file = try repo.openFile(io, filename, .{}); - defer file.close(io); - try parseSafetensors(allocator, io, registry, file); - } -} - -pub fn resolveFiletype(io: std.Io, file: std.Io.File) !FileType { - var path_buf: [1024]u8 = undefined; - const path_len = try file.realPath(io, &path_buf); - - if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors")) { - return .safetensors; - } else { - return .unknown; - } -} - pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo( - io: std.Io, - path: []const u8, -) ModelPathResolutionError!std.Io.Dir { - if (std.mem.endsWith(u8, path, ".safetensors.index.json") or - std.mem.endsWith(u8, path, ".safetensors")) - { - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { + switch (FileType.resolve(path)) { + .full_index, .index, .safetensors => { + // The path is a file, let's open it's folder + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - }; - return dir; + return dir; + }, + .unknown => { + // The path is probably a directory, let's try to open it as such + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return dir; + }, } - - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - - return dir; } -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!std.Io.File { - { - const index_name = "model.safetensors.index.json"; - if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { - return index_file; - } else |_| {} - } - - { - const model_name = "model.safetensors"; - if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { - return model_file; - } else |_| {} - } - - return ModelPathResolutionError.FileNotFound; -} - -const Metadata = union(enum) { - null: void, - int: i64, - float: f64, - bool: bool, - string: []const u8, - - array_bool: []const bool, - array_int: []const i64, - array_float: []const f64, - array_string: []const []const u8, - - pub const ItemType = enum { - int, - float, - bool, - string, - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .int => i64, - .float => f64, - .bool => bool, - .string => []const u8, - }; - } +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { + const default_entrypoints: []const struct { FileType, []const u8 } = &.{ + .{ .full_index, "model.safetensors.full_index.json" }, + .{ .full_index, "consolidated.safetensors.full_index.json" }, + .{ .index, "model.safetensors.index.json" }, + .{ .index, "consolidated.safetensors.index.json" }, + .{ .safetensors, "model.safetensors" }, + .{ .safetensors, "consolidated.safetensors" }, }; - pub fn wrap(x: anytype) Metadata { - return switch (@TypeOf(x)) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, - inline f16, f32, f64 => .{ .float = @floatCast(x) }, - bool => .{ .bool = x }, - []const u8 => .{ .string = x }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), - }; + for (default_entrypoints) |entrypoint| { + const file_type, const file_name = entrypoint; + const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; + + return .{ file_type, file }; } - pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { - return switch (@TypeOf(any_slice[0])) { - inline u8, i8, u16, i16, u32, i32, u64, i64 => { - const res = try allocator.alloc(i64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @intCast(val); - return .{ .array_int = res }; - }, - inline f16, f32, f64 => { - const res = try allocator.alloc(f64, any_slice.len); - for (res, any_slice) |*r, val| r.* = @floatCast(val); - return .{ .array_float = res }; - }, - bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, - []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, - else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), - }; - } + return error.FileNotFound; +} - pub fn format( - self: Metadata, - writer: *std.Io.Writer, - ) !void { - switch (self) { - .null => _ = try writer.write("null"), - .string => |s| try writer.print("{s}", .{s}), - .bool => |b| try writer.print("{}", .{b}), - .int => |i| try writer.print("{d}", .{i}), - .float => |f| try writer.print("{d}", .{f}), - .array_bool => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{}", .{v}); - } - try writer.writeByte(']'); - }, - .array_int => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_float => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("{d}", .{v}); - } - try writer.writeByte(']'); - }, - .array_string => |arr| { - try writer.writeByte('['); - for (arr, 0..) |v, i| { - if (i > 0) try writer.writeAll(", "); - try writer.print("\"{s}\"", .{v}); - } - try writer.writeByte(']'); - }, - } - } +const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, +}); - pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { - switch (self) { - .string => |s| allocator.free(s), - .array_bool => |s| allocator.free(s), - .array_int => |s| allocator.free(s), - .array_float => |s| allocator.free(s), - .array_string => |s| { - for (s) |str| allocator.free(str); - allocator.free(s); - }, - else => {}, - } - } - - pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { - return switch (self) { - .null => .null, - .int => |v| .{ .int = v }, - .float => |v| .{ .float = v }, - .bool => |v| .{ .bool = v }, - .string => |s| .{ .string = try allocator.dupe(u8, s) }, - .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, - .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, - .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, - .array_string => |s| blk: { - const new_slice = try allocator.alloc([]const u8, s.len); - errdefer allocator.free(new_slice); - for (s, 0..) |str, i| { - new_slice[i] = try allocator.dupe(u8, str); - } - break :blk .{ .array_string = new_slice }; - }, - }; - } -}; - -fn stringToDtype(safetensor_type: []const u8) !DataType { - const map = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); - - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); +fn stringToDtype(safetensor_dtype: []const u8) !DataType { + return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); return error.UnsupportedDataType; }; } - -fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { - var metadatas: Metadatas = .{}; - var prefix_buf: [1024]u8 = undefined; - var prefix = StringBuilder.initBuffer(&prefix_buf); - - try populateMetadata(allocator, &prefix, val, &metadatas); - - return metadatas; -} - -fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { - const key = prefix.items; - return switch (val) { - .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), - .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), - .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), - .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), - .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - if (validSlice(v)) |item_type| { - const data: Metadata = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ .array_bool = values }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ .array_int = values }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ .array_float = values }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); - } - break :blk .{ .array_string = values }; - }, - .null, .array, .object => unreachable, - }; - try metadatas.put(allocator, try allocator.dupe(u8, key), data); - } else { - for (v.items, 0..) |item, i| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try populateMetadata(allocator, prefix, item, metadatas); - prefix.items.len = old_len; - } - } - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - const old_len = prefix.items.len; - if (prefix.items.len > 0) { - prefix.appendAssumeCapacity('.'); - } - prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); - prefix.items.len = old_len; - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/shape.zig b/zml/shape.zig index 56c724e0..386b594e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -6,141 +6,26 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Sharding = @import("Sharding.zig"); const log = std.log.scoped(.shape); +test { + std.testing.refAllDecls(Shape); +} + /// Represent the shape of a tensor. +/// Contains also the sharding information, +/// so it describes exactly how the corresponding tensor/buffer will be layed out +/// across all devices of the current Platform. +/// +/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. +/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; - - pub const PartitionSpec = union(enum) { - axis: Tag, - open, - replicated, - unknown, - - pub fn init(comptime value: anytype) PartitionSpec { - const T = @TypeOf(value); - - if (T == PartitionSpec) { - return value; - } - - if (comptime T == @EnumLiteral()) { - const name = @tagName(value); - inline for (std.meta.fields(PartitionSpec)) |field| { - if (field.type == void and std.mem.eql(u8, field.name, name)) { - return @field(PartitionSpec, field.name); - } - } - } - - if (comptime isTagConvertible(T)) { - const tag_ = Shape.toTag(value); - - return .{ .axis = tag_ }; - } - - stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); - } - - test "PartitionSpec.init" { - const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; - try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); - - const spec_axis_open: PartitionSpec = .open; - try testing.expect(spec_axis_open.eql(.init(.open))); - } - - pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { - if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { - return false; - } - - switch (self) { - .axis => |t1| { - const t2 = other.axis; - return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); - }, - else => return true, - } - } - - test "PartitionSpec.eql" { - const spec_axis: PartitionSpec = .init(.a); - const spec_axis_b: PartitionSpec = .init(.b); - const spec_open: PartitionSpec = .open; - const spec_replicated: PartitionSpec = .replicated; - const spec_unknown: PartitionSpec = .unknown; - - try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); - try testing.expect(!spec_axis.eql(spec_axis_b)); - try testing.expect(!spec_axis.eql(.open)); - try testing.expect(!spec_axis.eql(.replicated)); - try testing.expect(!spec_axis.eql(.unknown)); - - try testing.expect(spec_open.eql(.open)); - try testing.expect(!spec_open.eql(spec_axis)); - try testing.expect(!spec_open.eql(.replicated)); - try testing.expect(!spec_open.eql(.unknown)); - - try testing.expect(spec_replicated.eql(.replicated)); - try testing.expect(!spec_replicated.eql(spec_axis)); - try testing.expect(!spec_replicated.eql(.open)); - try testing.expect(!spec_replicated.eql(.unknown)); - - try testing.expect(spec_unknown.eql(.unknown)); - try testing.expect(!spec_unknown.eql(spec_axis)); - try testing.expect(!spec_unknown.eql(.open)); - try testing.expect(!spec_unknown.eql(.replicated)); - } - - pub fn isClosed(self: PartitionSpec) bool { - return switch (self) { - .axis => true, - .open => false, - .replicated => true, - .unknown => false, - }; - } - - test isClosed { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.isClosed()); - - const spec_open: PartitionSpec = .open; - try testing.expect(!spec_open.isClosed()); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.isClosed()); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(!spec_unknown.isClosed()); - } - - pub fn toTag(self: PartitionSpec) Tag { - return switch (self) { - .axis => |t| return t, - .open => TagOpen, - .replicated => TagReplicated, - .unknown => TagUnknown, - }; - } - - test "PartitionSpec.toTag" { - const spec_axis: PartitionSpec = .init(.a); - try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); - - const spec_open: PartitionSpec = .open; - try testing.expect(spec_open.toTag() == TagOpen); - - const spec_replicated: PartitionSpec = .replicated; - try testing.expect(spec_replicated.toTag() == TagReplicated); - - const spec_unknown: PartitionSpec = .unknown; - try testing.expect(spec_unknown.toTag() == TagUnknown); - } - }; + _dtype: DataType, + _dims: DimsArray, + _tags: TagsArray, + _sharding: Sharding, pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -148,17 +33,10 @@ pub const Shape = struct { const TagReplicated = "replicated".ptr; const TagOpen = "open".ptr; + pub const MAX_RANK: u8 = constants.MAX_RANK; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); - pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; - - _dtype: DataType, - _dims: DimsArray = .{}, - _tags: TagsArray = UnknownTags, - _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -168,8 +46,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { - var dims_ = DimsArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; for (v) |d| { dims_.appendAssumeCapacity(@intCast(d)); tags_.appendAssumeCapacity(TagUnknown); @@ -178,21 +56,21 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - var dims_: DimsArray = .{}; - var tags_: TagsArray = .{}; - inline for (std.meta.fields(T)) |field| { - const fv = @field(v, field.name); - if (comptime stdx.meta.isInteger(field.type)) { + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + const fv = @field(v, field_name); + if (comptime stdx.meta.isInteger(Field)) { dims_.appendAssumeCapacity(@intCast(fv)); } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } @@ -217,8 +95,8 @@ pub const Shape = struct { return self.parseAxes(self.tags()); } - var axes_ = AxesArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + var axes_: AxesArray = .empty; + var tags_: TagsArray = .empty; if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { for (v) |d| { @@ -229,9 +107,9 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (std.meta.fields(T)) |field| { - axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); } return .{ axes_, tags_ }; } @@ -239,41 +117,69 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) TagsArray { + pub fn parseTags(v: anytype) []const Shape.Tag { const T = @TypeOf(v); - stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_ = TagsArray.init(0) catch unreachable; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); + if (T == []const Shape.Tag) return v; + comptime validateTags(T); + return comptime frozen: { + var tagz: [v.len]Shape.Tag = undefined; + for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; + // Force the creation of a comptime constant, so that we can return a pointer to it. + const frozen_tags = tagz; + break :frozen &frozen_tags; + }; + } + + fn validateTags(T: type) void { + const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); + switch (@typeInfo(T)) { + .@"struct" => |info| { + if (!info.is_tuple) @compileError(message); + for (info.field_types) |Field| { + if (Field != @EnumLiteral()) @compileError(message); + } + if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); + }, + else => @compileError(message), } - return tags_; } /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); + /// + /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); - - return res; + const _dims, const _tags = parseDimensions(dimz); + return .{ + ._dtype = dt, + ._dims = _dims, + ._tags = _tags, + ._sharding = .replicated, + }; } pub fn scalar(dt: DataType) Shape { - return .{ ._dtype = dt }; + return .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, + ._sharding = .replicated, + }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; + stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); + var res: Shape = .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, + ._sharding = .replicated, + }; for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; + res._dims.appendAssumeCapacity(@intCast(i)); } - res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -302,9 +208,7 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral() => true, - std.builtin.Type.StructField => true, - Tag => true, + @EnumLiteral(), [:0]const u8, Tag => true, else => false, }; } @@ -313,14 +217,16 @@ pub const Shape = struct { const T = @TypeOf(v); return switch (T) { @EnumLiteral() => @tagName(v).ptr, - std.builtin.Type.StructField => v.name.ptr, + [:0]const u8 => v.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), }; } - inline fn ensureAttributesAreSync(self: Shape) void { - stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); + fn ensureAttributesAreSync(self: Shape) void { + if (builtin.mode == .debug) { + stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + } } pub fn tag(self: Shape, ax: anytype) Tag { @@ -349,7 +255,7 @@ pub const Shape = struct { } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); + return axisFromTagMaybe(self.tags(), toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { @@ -387,15 +293,21 @@ pub const Shape = struct { self.ensureAttributesAreSync(); const T = @TypeOf(axis_); - if (comptime stdx.meta.isInteger(T)) { - return self.axisFromInt(@intCast(axis_)); - } - - if (comptime isTagConvertible(T)) { - return self.axisFromTag(toTag(axis_)); - } - - stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); + // Special handling of unsigned int, to remove runtime computations and favor inlining + return switch (@typeInfo(T)) { + .int => |info| if (info.signedness == .signed) + self.axisFromInt(axis_) + else + @intCast(axis_), + .comptime_int => if (axis_ < 0) + self.axisFromInt(axis_) + else + axis_, + else => if (comptime isTagConvertible(T)) + self.axisFromTag(toTag(axis_)) + else + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}), + }; } pub fn axes(self: Shape, axes_: anytype) AxesArray { @@ -407,7 +319,7 @@ pub const Shape = struct { return self.axes(axes_.tags()); } - var res = AxesArray.init(0) catch unreachable; + var res: AxesArray = .empty; if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { for (axes_) |ax| { @@ -417,8 +329,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); } return res; } @@ -426,7 +338,7 @@ pub const Shape = struct { stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); } - fn axisFromInt(self: Shape, a: isize) u3 { + pub fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); @@ -437,7 +349,7 @@ pub const Shape = struct { @intCast(a); } - fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { + pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { if (t == TagUnknown) return null; if (axisFromLiteralInt(t)) |ax| return ax; @@ -445,7 +357,7 @@ pub const Shape = struct { if (@inComptime()) { // At comptime two duplicated strings may have two different representations const t_bytes: []const u8 = std.mem.span(t); - for (self.tags(), 0..) |self_tag, ax| { + for (tagz, 0..) |self_tag, ax| { if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { return @truncate(ax); } @@ -454,7 +366,7 @@ pub const Shape = struct { } // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { + if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { return @truncate(ax); } return null; @@ -471,7 +383,7 @@ pub const Shape = struct { fn axisFromTag(self: Shape, d: Tag) u3 { stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { + return axisFromTagMaybe(self.tags(), d) orelse { stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -486,15 +398,15 @@ pub const Shape = struct { pub fn count(self: Shape) usize { var res: i64 = 1; for (self.dims()) |d| { - stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{&self}); res *= d; } return @intCast(res); } - /// Total size in bytes needed to represent this shape. + /// Total size in bytes needed to represent this shape on host pub fn byteSize(self: Shape) usize { - return self.dtype().sizeOf() * self.count(); + return std.math.divCeil(usize, self.count() * self.dtype().bitSizeOf(), 8) catch unreachable; } /// Compares the two shapes described, ignoring tagging. @@ -513,8 +425,7 @@ pub const Shape = struct { } /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" + /// "Shape({.a=10, .b=20}, dtype=.f32)" pub fn format(self: Shape, writer: *std.Io.Writer) !void { _ = try writer.writeByte('{'); @@ -528,9 +439,16 @@ pub const Shape = struct { try writer.print("{d}", .{d}); } - const part = self._partitioning.get(i); - if (part.toTag() != TagUnknown) { - try writer.print("/{s}", .{part.toTag()}); + const partitioning = self._sharding.get(i); + switch (partitioning) { + .replicated => {}, + .open => try writer.writeAll("/?"), + // this state is invalid, but don't panic cause we may be called from a panic + .out_of_bound => try writer.writeAll("/!"), + else => { + const mesh_ax = partitioning.meshAxis().?; + try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); + }, } need_comma = true; @@ -538,6 +456,10 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); _ = try writer.writeByte('}'); + if (self._sharding.mesh) |m| { + try writer.writeByte('@'); + try writer.writeAll(m.name); + } } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. @@ -563,16 +485,24 @@ pub const Shape = struct { return true; } + /// Reshape a given shape but keep the same total number of elements. + /// Partitioning information is generally NOT preserved, + /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); - new_shape.inferMissingAxis(self.count()); - stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); + const _dims, const _tags = parseDimensions(new_shape_); + var new_shape: Shape = .{ + ._dtype = self._dtype, + ._dims = _dims, + ._tags = _tags, + ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, + }; + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); + }; return new_shape; } - fn inferMissingAxis(self: *Shape, n_: usize) void { + fn inferMissingAxis(self: *Shape, n_: usize) error{ UnexpectedRemainder, TooBig }!void { stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; @@ -584,10 +514,10 @@ pub const Shape = struct { } } const n: i64 = @intCast(n_); + if (tmp_count > n) return error.TooBig; // Abort, `reshape` will panic with more context. - if (@mod(n, tmp_count) != 0) { - return; - } + if (@mod(n, tmp_count) != 0) return error.UnexpectedRemainder; + self._dims.set(inferred_ax, @divExact(n, tmp_count)); } @@ -605,6 +535,11 @@ pub const Shape = struct { const res = x.reshape(.{-1}); try testing.expectEqualSlices(i64, &.{30}, res.dims()); } + { + var y: Shape = .init(.{ 2, 5, -1 }, .f32); + try testing.expectError(error.UnexpectedRemainder, inferMissingAxis(&y, 15)); + try testing.expectError(error.TooBig, inferMissingAxis(&y, 8)); + } } pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { @@ -643,7 +578,7 @@ pub const Shape = struct { res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; for (0..dims_.len) |_| { - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); } return res; @@ -666,7 +601,7 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._partitioning.insert(ax, .unknown) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); return res; } @@ -676,7 +611,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); + for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); return res; } @@ -698,7 +633,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._partitioning.appendAssumeCapacity(.unknown); + res._sharding = res._sharding.set(self.rank(), .replicated); return res; } @@ -707,7 +642,7 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); - _ = res._partitioning.orderedRemove(a); + res._sharding = res._sharding.orderedRemove(a); return res; } @@ -724,6 +659,7 @@ pub const Shape = struct { std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); var sh: Shape = self; + sh._sharding.partition = .out_of_bound; const rk = self.rank(); var res_ax: u32 = 0; for (0..rk) |ax| { @@ -733,12 +669,11 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; + sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; - sh._partitioning.len = rk - to_remove.len; return sh; } @@ -761,12 +696,12 @@ pub const Shape = struct { var res = self; const original_dims = self._dims; const original_tags = self._tags; - const original_parts = self._partitioning; + const original_parts = self._sharding; for (permutations_.constSlice(), 0..) |permutation, i| { res._dims.set(i, original_dims.get(permutation)); res._tags.set(i, original_tags.get(permutation)); - res._partitioning.set(i, original_parts.get(permutation)); + res._sharding = res._sharding.set(i, original_parts.get(permutation)); } return res; @@ -785,9 +720,9 @@ pub const Shape = struct { ), ); - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); + try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); } /// Tag each ax of this shape with tags from a tuple. @@ -816,7 +751,7 @@ pub const Shape = struct { return res; } - stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + stdx.debug.compileError("Expected a tuple of enum literals eg: .{{ .a, .b, .c }} got: {any}", .{@TypeOf(tagz)}); } test withTags { @@ -890,160 +825,151 @@ pub const Shape = struct { } } + pub fn assertTags(self: Shape, tagz: anytype) void { + const T = @TypeOf(tagz); + if (T == Shape) { + return self.assertTags(tagz.tags()); + } else if (T != []const Shape.Tag) { + return self.assertTags(comptime parseTags(tagz)); + } + + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); + stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); + } + pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; return res; } - pub fn withPartitioning(self: Shape, specs: anytype) Shape { - const T = @TypeOf(specs); + // These tests only resolve logical names; placement does not use this fixture. + const test_mesh: Sharding.Mesh = .{ + .name = "test_mesh", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), + .bindings = .init(&.{ + .init(&.{.link_x}), + .init(&.{.link_y}), + .init(&.{.link_z}), + }), + .grid = undefined, + }; - var res = self.withDefaultPartitioning(); // todo add test for this new change - - if (stdx.meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - const partition_axis = @field(specs, field.name); - const axis_ = res.axisFromTagMaybe(toTag(field)); - - if (axis_) |ax| { - res._partitioning.set(ax, PartitionSpec.init(partition_axis)); - } else { - stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); - } - } - } else { - stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + /// Returns shape metadata with the requested sharding on `mesh`. + /// Omitted dimensions in a mapping are replicated, including during compilation. + /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. + pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { + if (@TypeOf(partitioning) != Sharding.Partitioning) { + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); } - - // Check that no mesh axis is used to partition multiple tensor dimensions. - var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; - - for (res._partitioning.constSlice()) |spec| { - const axis_tag = spec.toTag(); - - if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { - stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); - } - - switch (spec) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), - } + var res = self; + res._sharding = .{ .mesh = mesh, .partition = partitioning }; + for (0..self.rank()) |ax| { + if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); } + for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + + stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); return res; } test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); } - pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { + pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { const T = @TypeOf(mapping); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); - var res = self; - for (res._partitioning.slice()) |*spec| { - switch (spec.*) { - .unknown => {}, - .open => {}, - .replicated => {}, - .axis => |*old_axis_tag| { - inline for (std.meta.fields(T)) |field| { - const logical_axis_in_map = toTag(field); - - if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { - const physical_axis_from_map = @field(mapping, field.name); - spec.* = PartitionSpec.init(physical_axis_from_map); - break; - } - } - }, + for (0..self.rank()) |ax| { + const mesh_axis = self.partition(ax).meshAxis() orelse continue; + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); + if (mesh_axis == old_axis) { + const new_tag = toTag(@field(mapping, field_name)); + const new_axis = mesh.resolveLogicalAxis(new_tag) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); + res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); + break; + } } } return res; } test mapPartitioningAxes { - const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ - .batch = .data, - .seq = .replicated, - .features = .model, - }); - - const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); - - try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); - try testing.expect(physical_shape.partition(.features).eql(.init(.y))); - try testing.expect(physical_shape.partition(.seq).eql(.replicated)); - - const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); - try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); - try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); + const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) + .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); + const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); + try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); + const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); + try testing.expectEqual(shape.partition(.features), partial.partition(.features)); } pub fn withDefaultPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self.rank()); return res; } test withDefaultPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn withReplicatedPartitioning(self: Shape) Shape { var res = self; - res._partitioning.clear(); - res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); + res._sharding.partition = .replicated(self._dims.len); return res; } test withReplicatedPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); - shape = shape.withPartitioning(.{ ._1 = .batch }); - try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape._partitioning.len); - try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); } pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec == .axis) return true; + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (spec.isSharded()) return true; } return false; } test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); shape = shape.withDefaultPartitioning(); @@ -1051,20 +977,18 @@ pub const Shape = struct { } pub fn isFullyPartitioned(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - switch (spec) { - .replicated, .open, .unknown => return false, - .axis => {}, - } + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (!spec.isSharded()) return false; } return true; } test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); try testing.expect(shape.isFullyPartitioned()); - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyPartitioned()); shape = shape.withDefaultPartitioning(); @@ -1072,54 +996,47 @@ pub const Shape = struct { } pub fn isFullyReplicated(self: Shape) bool { - for (self._partitioning.constSlice()) |spec| { - if (spec != .replicated and spec != .unknown) { - return false; - } - } - return true; + return self._sharding.isFullyReplicated(); } test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); try testing.expect(shape.isFullyReplicated()); - shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); - try testing.expect(shape.isFullyReplicated()); - - shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); try testing.expect(!shape.isFullyReplicated()); } - pub fn partition(self: Shape, ax: anytype) PartitionSpec { - return self._partitioning.get(self.axis(ax)); + pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { + return self._sharding.get(self.axis(ax)); } test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); } - pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { - for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { - if (dim_part.eql(part)) return true; + pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { + for (0..self.rank()) |ax| { + const dim_part = self._sharding.get(ax); + if (dim_part == part) return true; } return false; } test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.init(.batch))); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.sharded(0))); try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); - try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.init(.batch))); - try testing.expect(shape.containsPartitionSpec(.unknown)); - try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.sharded(0))); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(shape.containsPartitionSpec(.replicated)); try testing.expect(!shape.containsPartitionSpec(.open)); } @@ -1129,10 +1046,10 @@ pub const Shape = struct { const T = @TypeOf(renames); stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - const new_field = @field(renames, field.name); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const new_field = @field(renames, field_name); stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field), toTag(new_field)); + res._tags.set(self.axis(field_name), toTag(new_field)); } return res; } @@ -1247,21 +1164,28 @@ pub const Shape = struct { } } + test "byte strides handle zero dimensions and maximum rank" { + const empty = Shape.init(.{ 2, 0, 5 }, .f32).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 0, 20, 4 }, empty.constSlice()); + const full_rank = Shape.init(.{ 2, 3, 5, 7, 11, 13, 17, 19 }, .u8).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4849845, 1616615, 323323, 46189, 4199, 323, 19, 1 }, full_rank.constSlice()); + const large = Shape.init(.{std.math.maxInt(i64)}, .f64).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{8}, large.constSlice()); + } + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { + @setRuntimeSafety(false); const rk = self.rank(); - var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; - if (rk == 0) return strides; + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .buffer = undefined, .len = rk }; + var stride = element_size; + var ax: usize = rk; + while (ax > 0) { + ax -= 1; + strides.buffer[ax] = stride; + // Axis 0 is not part of any stride; avoid computing the total size. + if (ax > 0) stride *= self._dims.buffer[ax]; + } - const V = @Vector(constants.MAX_RANK, i64); - const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); - // For each axis compute the product of all following dimensions - // and the element size in bytes. - var d: V = @bitCast(self._dims.buffer); - d = @select(i64, rank_mask, d, @as(V, @splat(1))); - d = std.simd.shiftElementsLeft(d, 1, element_size); - d = std.simd.prefixScan(.Mul, -1, d); - - strides.buffer = @bitCast(d); return strides; } @@ -1269,7 +1193,7 @@ pub const Shape = struct { /// so that the given axes are contiguous. pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { const axes__, _ = self.parseAxes(axes_); - var perms = AxesArray.init(0) catch unreachable; + var perms: AxesArray = .empty; for (0..self.rank()) |i| { if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { continue; @@ -1317,26 +1241,27 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - _ = new_shape._partitioning.orderedRemove(ax); + new_shape._sharding = new_shape._sharding.orderedRemove(ax); for (0..dims_.len) |_| { - new_shape._partitioning.insert(ax, .unknown) catch unreachable; + new_shape._sharding = new_shape._sharding.insert(ax, .replicated); } - new_shape.inferMissingAxis(self.count()); + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't split {any} along axis {d} into {any}: {t}", .{ self.dims(), ax, new_shape.dims(), err }); + }; + return new_shape; } test splitAxis { - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = 5, .a2 = 2 }), ); - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = .auto, .a2 = 2 }), ); } @@ -1345,8 +1270,8 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - res = res.splitAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res = res.splitAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1370,39 +1295,34 @@ pub const Shape = struct { const axes_to_merge = self.axes(axes_); stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - var new_dim: i64 = 1; // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting sharding is ambiguous. A propagation pass must resolve this. + // the resulting mesh is ambiguous. A propagation pass must resolve this. // Marking it 'open' is the correct signal. If all are replicated/unknown, the // result is also replicated/unknown. - // rule: .replicated wins over .unknown. .open wins over everything. - var resulting_partition_spec: PartitionSpec = .unknown; - - for (0..axes_to_merge.len) |i| { - const current_axis = axes_to_merge.get(i); - new_dim *= self.dim(current_axis); - - if (i > 0) { - stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); - } - - switch (self.partition(current_axis)) { - .axis, .open => resulting_partition_spec = .open, - .replicated => if (resulting_partition_spec != .open) { - resulting_partition_spec = .replicated; - }, - .unknown => {}, - } - } + // rule: .replicated wins over .out_of_bound. .open wins over everything. const first_axis_to_merge = axes_to_merge.get(0); const num_axes_to_merge = axes_to_merge.len; - var new_shape = self; + var merged_dim: i64 = 1; + var merged_spec: Sharding.PartitionSpec = .out_of_bound; + for (axes_to_merge.constSlice()) |ax| { + merged_dim *= self.dim(ax); + const spec = self.partition(ax); + if (spec.isSharded() or spec == .open) { + merged_spec = .open; + } else if (spec == .replicated and merged_spec != .open) { + merged_spec = .replicated; + } + } - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); + var new_shape = self; + for (1..num_axes_to_merge) |_| { + new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); + } + new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); - new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); return new_shape; } @@ -1438,23 +1358,23 @@ pub const Shape = struct { test "mergeAxis partitioning" { // Merging a partitioned axis with a replicated one -> open const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab).eql(.open)); + try testing.expect(merged1.partition(.ab) == .open); try testing.expect(merged1.dim(.ab) == 10); try testing.expect(merged1.rank() == 2); // Merging two replicated axes -> replicated const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab).eql(.replicated)); - try testing.expect(merged2.partition(.c).eql(.init(.y))); + try testing.expect(merged2.partition(.ab) == .replicated); + try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab).eql(.unknown)); + try testing.expect(merged3.partition(.ab) == .replicated); } pub fn mergeAxes(self: Shape, axes_: anytype) Shape { @@ -1462,9 +1382,9 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); - res = res.mergeAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); + res = res.mergeAxis(field_name, @field(axes_, field_name)); } return res; } @@ -1493,7 +1413,7 @@ pub const Shape = struct { } fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { - var res = TagsArray.init(0) catch unreachable; + var res: TagsArray = .empty; for (a) |tag_| { if (std.mem.indexOfScalar(Tag, b, tag_)) { res.appendAssumeCapacity(tag_); @@ -1505,8 +1425,8 @@ pub const Shape = struct { pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; - var tags_: TagsArray = .{}; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; + var tags_: TagsArray = .empty; if (comptime stdx.meta.isSliceOf(V, T)) { for (v) |d| { @@ -1516,14 +1436,14 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(V)) { - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); - inline for (fields) |field| { - const fv = @field(v, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); vals_.appendAssumeCapacity(fv); if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } return .{ vals_, tags_ }; @@ -1543,7 +1463,7 @@ pub const Shape = struct { pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1553,11 +1473,11 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const fields = std.meta.fields(V); - stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); - inline for (fields) |field| { - const a = self.axis(field); - res.buffer[a] = @field(options, field.name); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); + inline for (field_names) |field_name| { + const a = self.axis(field_name); + res.buffer[a] = @field(options, field_name); } return res; } @@ -1608,10 +1528,57 @@ pub const Shape = struct { pub fn iterator(self: Shape) MultiDimIterator { return .{ .shape = self, - .current_coords = .{0} ** Shape.MAX_RANK, + .current_coords = @splat(0), .flat_index = 0, }; } + + /// Generate the packed shape visible to PJRT + pub fn packedShape(sh: Shape) Shape { + const bit_size = sh.dtype().bitSizeOf(); + if (bit_size >= 8) return sh; + + const items_per_byte = @divExact(8, bit_size); + var res: Shape = sh.setDim(-1, @divExact(sh.dim(-1), items_per_byte)); + res._dtype = .u8; + return res; + } + + test packedShape { + const x: Shape = .init(.{ 4, 8 }, .u2); + try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); + } + + test "shape transformations preserve partition specs and clear unused slots" { + var shape = Shape.init(.{ 2, 3, 4 }, .f32); + shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; + for ([_]Shape{ + shape.insert(1, .{5}), + shape.append(.{5}), + shape.remove(1), + shape.removeMany(.{ 0, 1 }), + shape.transpose(.{ 2, 0, 1 }), + shape.reshape(.{24}), + shape.splitAxis(1, .{ 1, 3 }), + shape.mergeAxis(.merged, .{ 0, 1 }), + shape.withDefaultPartitioning(), + }) |transformed| { + try testing.expectEqual(&test_mesh, transformed._sharding.mesh); + } + const parts = shape._sharding.partition; + try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); + try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); + try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); + const removed = shape.removeMany(.{ 0, 1 }); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); + try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); + } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional @@ -1736,3 +1703,10 @@ test "MultiDimIterator / Shape with dimension of size 1" { } try std.testing.expectEqual(6, count); } + +fn expectEqualShapes(expected: Shape, actual: Shape) error{TestExpectedEqual}!void { + if (expected.eqlWithTags(actual)) return; + + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); + return error.TestExpectedEqual; +} diff --git a/zml/slice.zig b/zml/slice.zig index e33316db..9451165b 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -2,9 +2,11 @@ const std = @import("std"); const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; +const builtin = @import("builtin"); pub fn isBytes(comptime T: type) bool { const type_info = @typeInfo(T); @@ -17,20 +19,35 @@ pub fn isBytes(comptime T: type) bool { } pub const Slice = struct { - inner_data: union(enum) { - mutable: []u8, - immutable: []const u8, - }, + bytes: []const u8, + mutable: bool, shape: Shape, + offset_bytes: usize = 0, + byte_strides: stdx.BoundedArray(i64, constants.MAX_RANK), pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); - return if (type_info.pointer.is_const) - .{ .inner_data = .{ .immutable = bytes }, .shape = shape } - else - .{ .inner_data = .{ .mutable = bytes }, .shape = shape }; + const byte_strides = shape.computeByteStrides(); + return .{ + .bytes = bytes, + .mutable = !type_info.pointer.attrs.@"const", + .shape = shape, + .offset_bytes = 0, + .byte_strides = byte_strides, + }; + } + + pub fn initConst(shape: Shape, bytes: []const u8) Slice { + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); + return .{ + .bytes = bytes, + .mutable = false, + .shape = shape, + .offset_bytes = 0, + .byte_strides = shape.computeByteStrides(), + }; } pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { @@ -46,12 +63,18 @@ pub const Slice = struct { else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), }; - return .{ .inner_data = .{ .mutable = bytes }, .shape = shape_ }; + return .{ + .bytes = bytes, + .mutable = true, + .shape = shape_, + .offset_bytes = 0, + .byte_strides = shape_.computeByteStrides(), + }; } pub fn free(slice: Slice, allocator: std.mem.Allocator) void { - const d = slice.constData(); - + const d = slice.bytes; + // TODO: It would be better to store the aligment, this can be error prone in the presence of type casting switch (slice.shape.dtype().alignOf()) { 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), 2 => allocator.free(@as([]align(2) const u8, @alignCast(d))), @@ -68,32 +91,158 @@ pub const Slice = struct { return slice.shape.dtype(); } + pub fn isContiguous(slice: Slice) bool { + const expected = slice.shape.computeByteStrides(); + for (expected.slice(), slice.byte_strides.slice()) |exp, actual| { + if (exp != actual) return false; + } + return true; + } + + /// Returns the actual element-wise strides for this specific slice. + /// This respects transpositions and non-contiguous views. + pub fn elementStrides(self: Slice) stdx.BoundedArray(i64, constants.MAX_RANK) { + const element_size = @as(i64, @intCast(self.dtype().sizeOf())); + var res = self.byte_strides; + for (res.slice()) |*stride| { + // We divide the actual byte strides by the size of the data type + stride.* = @divExact(stride.*, element_size); + } + return res; + } + + pub fn subSlice(slice: Slice, axis: anytype, start: i64, size: i64) Slice { + const dim = slice.shape.dim(axis); + + stdx.debug.assert(start >= 0 and size >= 0 and start + size <= dim, "subSlice out of bounds: axis {d} start {d} size {d} dim {d}", .{ axis, start, size, dim }); + + const stride_bytes = slice.byte_strides.get(axis); + const new_offset = @as(i64, @intCast(slice.offset_bytes)) + start * stride_bytes; + stdx.debug.assert(new_offset >= 0, "subSlice produced negative offset: {d}", .{new_offset}); + + var res = slice; + res.offset_bytes = @intCast(new_offset); + res.shape = slice.shape.set(axis, size); + return res; + } + + pub fn squeeze(slice: Slice, axis_: anytype) Slice { + const axis = slice.shape.axis(axis_); + var res = slice; + res.shape = slice.shape.drop(axis); + _ = res.byte_strides.orderedRemove(@intCast(axis)); + return res; + } + pub fn data(slice: Slice) []u8 { - return switch (slice.inner_data) { - .mutable => |d| d, - else => stdx.debug.panic("Expected slice to be mutable but it's immutable", .{}), - }; + std.debug.assert(slice.mutable); + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return @constCast(slice.bytes[slice.offset_bytes..end]); } pub fn constData(slice: Slice) []const u8 { - return switch (slice.inner_data) { - inline else => |d| d, - }; + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return slice.bytes[slice.offset_bytes..end]; } pub fn items(slice: Slice, comptime T: type) []T { - return @alignCast(std.mem.bytesAsSlice(T, slice.data())); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); + return @ptrCast(@alignCast(slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - return @alignCast(std.mem.bytesAsSlice(T, slice.constData())); + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); + return @ptrCast(@alignCast(slice.constData())); + } + + pub fn contiguousItemsIterator(slice: *const Slice, comptime T: type) ContiguousItemsIterator(T) { + std.debug.assert(slice.dtype() == DataType.fromZigType(T)); + return .init(slice); + } + + fn ContiguousItemsIterator(comptime T: type) type { + return struct { + slice: *const Slice, + contiguous_axis_start: u4, + contiguous_item_count: usize, + outer_index: usize, + outer_count: usize, + outer_coords: [constants.MAX_RANK]usize, + next_offset_bytes: usize, + + const Iterator = @This(); + + fn init(slice: *const Slice) Iterator { + const rank = slice.shape.rank(); + const total_count = slice.shape.count(); + var contiguous_axis_start: u4 = rank; + var contiguous_item_count: usize = 1; + var expected_stride: i64 = @intCast(@sizeOf(T)); + + var axis = rank; + while (axis > 0) { + axis -= 1; + const dim = slice.shape.dim(axis); + const stride = slice.byte_strides.get(axis); + if (dim != 1 and stride != expected_stride) break; + contiguous_axis_start = axis; + contiguous_item_count *= @intCast(dim); + expected_stride *= dim; + } + + if (total_count == 0) { + contiguous_item_count = 0; + } + + return .{ + .slice = slice, + .contiguous_axis_start = contiguous_axis_start, + .contiguous_item_count = contiguous_item_count, + .outer_index = 0, + .outer_count = if (contiguous_item_count == 0) 0 else total_count / contiguous_item_count, + .outer_coords = @splat(0), + .next_offset_bytes = slice.offset_bytes, + }; + } + + pub fn next(self: *Iterator) ?[]const T { + if (self.outer_index >= self.outer_count) return null; + + const start = self.next_offset_bytes; + self.outer_index += 1; + if (self.outer_index < self.outer_count) { + self.advanceOuterCoords(); + } + + const byte_len = self.contiguous_item_count * @sizeOf(T); + return @ptrCast(@alignCast(self.slice.bytes[start..][0..byte_len])); + } + + fn advanceOuterCoords(self: *Iterator) void { + var axis = self.contiguous_axis_start; + while (axis > 0) { + axis -= 1; + + self.outer_coords[axis] += 1; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) + self.slice.byte_strides.get(axis)); + + const dim: usize = @intCast(self.slice.shape.dim(axis)); + if (self.outer_coords[axis] < dim) return; + + self.outer_coords[axis] = 0; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) - @as(i64, @intCast(dim)) * self.slice.byte_strides.get(axis)); + } + } + }; } pub fn format( slice: @This(), writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - return writer.print("{any}", .{slice}); + return writer.print("Slice({f})@0x{x} [mut={}, off={d}, strides={any}]", .{ slice.shape, @intFromPtr(slice.bytes.ptr), slice.mutable, slice.offset_bytes, slice.byte_strides.slice() }); } pub fn formatNumber(slice: Slice, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { @@ -109,7 +258,8 @@ pub const Slice = struct { // Special case input tensor is a scalar return switch (slice.dtype()) { inline else => |dt| { - const val: dt.toZigType() = slice.constItems(dt.toZigType())[0]; + const single_ptr: *const dt.toZigType() = @ptrCast(@alignCast(slice.bytes.ptr)); + const val = single_ptr.*; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), @@ -127,15 +277,28 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); switch (slice.dtype()) { inline else => |dt| { - const T = dt.toZigType(); - // TODO: handle negative strides - const elem_strides: i64 = slice.shape.computeElementStrides().get(0); - const values = slice.constItems(T); + const T = dt.toPackedZigType(); + const n = slice.shape.dim(0); + const elem_per_bytes: comptime_int = @max(1, 8 / comptime dt.bitSizeOf()); + const stride = @divExact(slice.byte_strides.get(0), @as(i64, @sizeOf(T))); + + // Fetch n next elements, accounting for strides and packing + const needed_len: usize = @as(usize, if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1)) / elem_per_bytes; + + // The formatter consumes this physical tail using `stride`, so this + // rank-1 path can read raw items without requiring contiguity. + const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); + const values = raw_values[0..needed_len]; + + if (comptime dt.bitSizeOf() < 8) { + return try stdx.fmt.formatSliceAny(values, options, stride, writer); + } + switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, stride, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, stride, writer), } }, } @@ -179,8 +342,83 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); _ = try writer.write("}"); } + + pub fn copy(destination: Slice, source: []const u8) void { + const total_bytes = destination.shape.byteSize(); + stdx.debug.assert(source.len >= total_bytes, "Source buffer size ({}) smaller than destination shape ({})", .{ source.len, total_bytes }); + + if (total_bytes == 0) return; + + if (destination.isContiguous()) { + @memcpy(destination.data()[0..total_bytes], source[0..total_bytes]); + return; + } + + const rank = destination.shape.rank(); + stdx.debug.assert(rank > 0, "Non-contiguous slice with rank 0 is unexpected", .{}); + + const outer_len: usize = @intCast(destination.shape.dim(0)); + stdx.debug.assert(outer_len > 0, "Invalid dimension size: {d}", .{outer_len}); + + stdx.debug.assert(total_bytes % outer_len == 0, "Non-even split for contiguous source", .{}); + const outer_span_bytes = total_bytes / outer_len; + + for (0..outer_len) |i| { + const destination_outer = destination.subSlice(0, @intCast(i), 1).squeeze(0); + const source_outer_start = i * outer_span_bytes; + const source_outer_end = source_outer_start + outer_span_bytes; + destination_outer.copy(source[source_outer_start..source_outer_end]); + } + } }; +test "slice expectClose compares dense and strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 2, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, .exact_match); + try testing.expectClose(std.testing.io, strided, dense, .exact_match); +} + +test "slice expectClose applies close fraction across strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 3, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + const opts: testing.CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .minimum_close_fraction = 0.75, + }; + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, opts); + try testing.expectClose(std.testing.io, strided, dense, opts); +} + test "slice pretty print rank 3" { const data: [4][4][4]i32 = .{ .{ @@ -266,6 +504,14 @@ test "slice pretty print rank 0" { try std.testing.expectFmt(expected, "{d}", .{slice}); } +test "slice pretty print bool" { + const data: [2]bool = .{ true, false }; + const slice = Slice.init(.init(.{2}, .bool), std.mem.asBytes(&data)); + const expected = "{1,0}"; + try std.testing.expectEqualSlices(bool, &data, slice.constItems(bool)); + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + test "slice pretty print ellipsis" { const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; const slice = Slice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); @@ -284,3 +530,12 @@ test "slice pretty print ellipsis" { ; try std.testing.expectFmt(expected, "{d}", .{slice}); } + +test "slice pretty print f4" { + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + const slice = Slice.initConst(.init(.{8}, .f4e2m1), @ptrCast(&x_f4_packed)); + + const expected = "{0,0.5,1,1.5,2,3,4,6}"; + + try std.testing.expectFmt(expected, "{d}", .{slice}); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 1b2f8736..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1,36 +1,49 @@ const std = @import("std"); const builtin = @import("builtin"); -const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("zml.zig").Bufferized; -const CompilationContext = @import("module.zig").CompilationContext; +const Compiler = @import("Compiler.zig"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Tensor); +} pub const Tensor = struct { - var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; + pub const Id = enum(u64) { _ }; + var current_id: std.atomic.Value(u64) = .init(1); - id: usize, + id: Tensor.Id, auto_broadcast: bool = false, _shape: Shape, _value: ?*const mlir.Value = null, + const ResolvedAxis = u3; + pub fn init(shape_like: anytype, dt: DataType) Tensor { return .fromShape(.init(shape_like, dt)); } + fn nextTensorId() Id { + return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); + } + pub fn fromShape(shape_: Shape) Tensor { - return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; + return .{ .id = nextTensorId(), ._shape = shape_ }; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -62,42 +75,50 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. + /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, + /// and open sharding for the partitioner to infer from graph constraints. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - const res: Tensor = .{ - ._shape = sh, - ._value = val, - .id = Tensor.current_id.fetchAdd(1, .seq_cst), - }; + var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + res._shape._sharding = .open; - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); - stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + const eql_dtype = sh.dtype() == other.dtype(); + const eql_dims = std.mem.eql(i64, sh.dims(), other.dims()); + stdx.debug.internalAssert(eql_dims and eql_dtype, "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); } return res; } - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. + /// Internal: creates a constructor result and honors `sh`'s partitioning when + /// it specifies a mesh. + pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { + const result = _result(sh, val); + const mesh = sh._sharding.mesh orelse return result; + return result.withPartitioning(mesh, sh._sharding.partition); + } + + /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. + /// Dimension tags are unknown and sharding is open for partitioner inference. pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; const n = ranked_tensor.rank(); stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; + var sh: Shape = .{ + ._dtype = ctx.dtype(ranked_tensor.elementType()), + ._dims = .empty, + ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, + ._sharding = .open, + }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.resize(n) catch unreachable; - sh._partitioning.resize(n) catch unreachable; - - return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; + return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } /// Returns the dimension of axis 'axis_'. @@ -115,14 +136,14 @@ pub const Tensor = struct { /// Returns the index of axis 'axis_'. /// /// 'axis_' can be an integer or a tag. - pub fn axis(self: Tensor, axis_: anytype) u3 { + pub fn axis(self: Tensor, axis_: anytype) ResolvedAxis { return self._shape.axis(axis_); } /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(ResolvedAxis, constants.MAX_RANK) { return self._shape.axes(axes_); } @@ -142,25 +163,144 @@ pub const Tensor = struct { return res; } - // TODO(Corentin) - pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - _ = self; - _ = axes_; - @panic("Unimplemented"); + /// Declares this tensor's sharding on `mesh_`. + /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. + /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension + /// tags to logical mesh axes, `.replicated`, or `.open`. + /// + /// During compilation, emits a graph constraint. For example, + /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` + /// and leaves omitted dimensions open for inference. Use `.replicated` to + /// keep a dimension whole on each shard. + /// + /// Outside compilation, updates the tensor's shape to describe an executable + /// input's sharding. Omitted dimensions are replicated and no graph constraint is + /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. + /// + /// Executable inputs and explicitly annotated outputs must have no open + /// dimensions. An output with no declared sharding defaults to replication. + pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { + if (@TypeOf(mesh_) == @EnumLiteral()) { + const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); + return self.withPartitioning(compiler.mesh(mesh_), partitioning); + } + + if (@TypeOf(partitioning) != Sharding.Partitioning) { + const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); + return self.withPartitioning(mesh_, parsed); + } + + const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); + + const ctx = Compiler.currentOrNull() orelse { + var res = self; + res._shape = partitioned_shape; + return res; + }; + + const attr = ctx.tensorShardingAttr(ctx.allocator, partitioned_shape) catch @panic("OOM"); + + const op_result = switch (ctx.partitioner) { + .shardy => blk: { + const op = shardy.sharding_constraint( + ctx.mlir_ctx, + self.value(), + attr.isA(shardy.TensorShardingAttr).?, + ctx.location, + ).appendTo(currentBlock()); + break :blk op.result(0); + }, + .gspmd => blk: { + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "Sharding", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.sharding", attr), + }, + }, + ctx.location, + ).appendTo(currentBlock()); + break :blk op.result(0); + }, + }; + + var res = _result(partitioned_shape, op_result); + res._shape = partitioned_shape; + return res; } - pub fn toMemory(self: Tensor, kind: Memory) Tensor { - const ctx = CompilationContext.current(); - if (ctx.platform.target == .cpu) return self; + /// Emits a graph constraint requiring a complete copy on every device. + /// Requires active compilation. For a replicated input or buffer description, + /// use the default sharding from `Tensor.init` or `Shape.init`. + pub fn replicate(self: Tensor) Tensor { + const compiler = Compiler.current(); + const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); + return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); + } - const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute( + test withPartitioning { + // Create two similar meshes, + // Then check that tensor.withPartitioning don't confuse the meshes + const dp_mp: Sharding.Mesh = .{ + .name = "dp_mp", + .physical = undefined, + .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .grid = undefined, + }; + + const mp_dp: Sharding.Mesh = .{ + .name = "mp_dp", + .physical = undefined, + .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .grid = undefined, + }; + + const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); + try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); + try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); + try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); + try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); + try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); + } + + /// Copy the given tensor to the specified memory, preserving its sharding. + pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ + .named( ctx.mlir_ctx, - ctx.platform.memoryKind(kind), - )), + "_xla_buffer_placement", + .string(ctx.mlir_ctx, ctx.platform.memoryKind(kind)), + ), }); - const op = dialects.stablehlo.custom_call( + const op = stablehlo.custom_call( ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, @@ -172,14 +312,97 @@ pub const Tensor = struct { .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), }, }, - .unknown(ctx.mlir_ctx), + ctx.location, ).appendTo(currentBlock()); var res = _result(self._shape, op.result(0)); - ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; + res._shape._sharding = self._shape._sharding; + ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); return res; } + test toMemory { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; + + const inputs: [8]f32 = .{ -3.0, -2, -1, 1, 2, 3, 5, -5 }; + const x_t = Tensor.init(.{8}, .f32); + + const Local = struct { + fn memcpyH2D(x: Tensor) Tensor { + return x.onMemory(.host_pinned).toMemory(.device); + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); + defer exe.deinit(); + + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); + defer x_h.deinit(); + + const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); + try std.testing.expectEqualSlices(f32, &inputs, x_h_ptr[0..8]); + + var x_d = try exe.eval(std.testing.allocator, io, .{x_h}); + defer x_d.deinit(); + + try zml.testing.expectClose(std.testing.io, x_h, x_d, .exact_match); + } + + /// Copy all the given tensor to the specified memory. + /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. + pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => return flat_tensors, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + var copy = flat_tensors; + meta.visitFlatStruct(struct { + fn toMemory(k: Memory.Kind, x: *Tensor) void { + x.* = x.toMemory(k); + } + }.toMemory, kind, ©); + return copy; + } + + /// Mark the given input tensor as being physically located on a specific memory. + /// Has no effect if the input tensor is not an executable input. + pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => return self, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + if (ctx.currentScope().id_to_argument.get(self.id) == null) { + return self; + } + + ctx.currentScope().id_to_memory.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; + + return self; + } + + /// Mark all the given input tensors as being physically located on a specific memory + /// see `zml.Tensor.onMemory` + pub fn onMemoryAll(tensors: anytype, kind: Memory.Kind) void { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + // Only one memory kind on those platform + .cpu, .neuron, .metal => return, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + meta.visit(struct { + fn onMemory(k: Memory.Kind, x: *const Tensor) void { + _ = x.onMemory(k); + } + }.onMemory, kind, &tensors); + } + /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -201,13 +424,12 @@ pub const Tensor = struct { /// Returns the mlir.Value associated with the Tensor. /// - /// This will fail if used outside of a compilation context. + /// This will fail if used outside of a Compiler context. pub fn value(self: Tensor) *const mlir.Value { - if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { - return CompilationContext.current().currentScope().block.argument(argument_index); - } else if (self._value) |v| { - return v; - } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + const scope = Compiler.current().currentScope(); + return scope.id_to_argument.get(self.id) orelse + self._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } /// Tell PJRT compiler that memory should be reuse between the two tensors. @@ -215,24 +437,108 @@ pub const Tensor = struct { /// but this API allows to reuse buffer between input and output arguments /// of a given function. /// Note this is visible from the outside. The caller of a function with donations - /// is not allowed to reuse the donated input buffer after the call. + /// cannot use the donated input's contents after execution. Runners destroy its + /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + /// + /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - const compilation_context = CompilationContext.current(); + const compilation_context = Compiler.current(); const scope = compilation_context.currentScope(); - if (scope.id_to_argument.get(origin.id)) |argument_index| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = argument_index; - } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { - const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; - gop.value_ptr.* = origin_donation; + if (scope.id_to_donation.get(origin.id)) |og_donation| { + // reuseBuffer is making the donation explicit + const donation: Compiler.Donation = switch (og_donation) { + .implicit => |input| .{ .explicit = input }, + .explicit => og_donation, + }; + scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); + + // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` + // We can now look at its sharding. + if (origin._shape.isFullyReplicated()) { + stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); + } else { + // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. + // We are stricter here and require the same shape. + stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); + + // Normally self sharding is unspecified and we can just propagate origin sharding, + // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. + stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); + } + + var res = self; + res._shape._sharding = origin._shape._sharding; + return res; + } else { + // Origin tensor doesn't has any physical buffer associated so we can move on. + // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, + // even though the caller may have constructed the inputs internally. + return self; + } + } + + test reuseBuffer { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; + + const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; + const x_t = Tensor.init(.{ 6, 2 }, .i4); + const y_t = Tensor.init(.{ 6, 2 }, .i4); + const z_t = Tensor.init(.{ 6, 2 }, .i4); + + const Local = struct { + fn memcopy(x: Tensor, y: Tensor, z: Tensor) [2]Tensor { + // the first output reuses the given y, but the second ouput does NOT reuse z. + return .{ x.addConstant(1).reuseBuffer(y), z }; + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); + defer exe.deinit(); + + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); + defer x_d.deinit(); + + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); + defer y_d.deinit(); + + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); + defer z_d.deinit(); + + var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + y_memory[dev] = y_d.opaqueDevicePtr(dev); + } + + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + + for ([_]bool{ false, true }) |wait| { + var z2_d: zml.Buffer = undefined; + runner.run(io, .{ x_d, y_d, z_d }, .{ &y_d, &z2_d }, .{ .wait = wait }); + defer z2_d.deinit(); + try y_d.await(io); + + var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + output_memory[dev] = y_d.opaqueDevicePtr(dev); + } + + // Check that the replacement y_d reuses the donated device memory. + try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y_d.numShards()]); + + // Check that z_d and z2_d DO NOT POINT to the same addresses + for (0..z2_d.numShards()) |dev| { + try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); + } } - return self; } /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -263,11 +569,11 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); - const op = dialects.stablehlo.bitcast_convert( + const op = stablehlo.bitcast_convert( mlirCtx(), self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); @@ -280,26 +586,26 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -321,31 +627,39 @@ pub const Tensor = struct { inline for (inputs, expectations, divisors) |i, e, d| { const input: zml.Tensor = .init(.{6}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.fmod, + .{ input, d }, + platform, + .{}, + ); defer exe.deinit(); var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); - try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); + try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, .{}); } } /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); + return binaryOp("shiftLeft", stablehlo.shift_left)(self, other); } /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); + return binaryOp("shiftRightArithmetic", stablehlo.shift_right_arithmetic)(self, other); } /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); + return binaryOp("shiftRightLogical", stablehlo.shift_right_logical)(self, other); } /// Returns the Cholesky decomposition of the input Tensor. @@ -355,24 +669,24 @@ pub const Tensor = struct { pub fn cholesky(self: Tensor, lower: bool) Tensor { stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { + pub fn triangularSolve(self: Tensor, other: Tensor, opts: stablehlo.TriangularSolveOpts) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); - stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); + stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); - const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(self._shape, op.result(0)); + const op = stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); + return _result(other._shape, op.result(0)); } /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -380,7 +694,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -389,7 +703,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -409,7 +723,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -430,12 +744,12 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { + pub fn fft(self: Tensor, opts: stablehlo.FftOpts) Tensor { // TODO: support tagged API. stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); @@ -455,41 +769,78 @@ pub const Tensor = struct { .f32 => .c64, else => .c128, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); + const shape_ = self._shape.setDim(-1, rfftOutputLen(self.dim(-1))); break :blk shape_.withDtype(dt); }, .IRFFT => blk: { stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + const fft_dims = self.dims()[self.rank() - opts.length.len ..]; + const fft_last_dim_index = opts.length.len - 1; + stdx.debug.assert(std.mem.eql(i64, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index]), "fft({any}) expects non-innermost tensor FFT dimensions to match given lengths, got {any} and {any}", .{ opts, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index] }); + stdx.debug.assert(self.dim(-1) == rfftOutputLen(opts.length[fft_last_dim_index]), "fft({any}) expects IRFFT innermost dimension to match the RFFT output length, got {} for FFT length {}", .{ opts, self.dim(-1), opts.length[fft_last_dim_index] }); const dt: DataType = switch (self.dtype()) { .c64 => .f32, else => .f64, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); + const shape_ = self._shape.setDim(-1, opts.length[fft_last_dim_index]); break :blk shape_.withDtype(dt); }, }; - const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(sh, op.result(0)); } + fn rfftOutputLen(input_len: i64) i64 { + const rfft_len_divisor = 2; + const rfft_edge_bin_count = 1; + return @divTrunc(input_len, rfft_len_divisor) + rfft_edge_bin_count; + } + + test "Tensor.fft IRFFT restores RFFT last dimension" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const rfft_input_len: i64 = 10; + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const freq = x.fft(.{ .kind = .RFFT, .length = &.{rfft_input_len} }); + std.debug.assert(freq.dim(-1) == rfftOutputLen(rfft_input_len)); + const time = freq.fft(.{ .kind = .IRFFT, .length = &.{rfft_input_len} }); + std.debug.assert(time.dim(-1) == x.dim(-1)); + return time; + } + }; + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{rfft_input_len}, .f32)}, + platform, + .{}, + ); + defer exe.deinit(); + } + pub const Rng = struct { _state: Tensor, - algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, + algorithm: stablehlo.RngAlgorithmAttr.Type = .DEFAULT, + + pub const Buffer = mem.Bufferized(Rng); pub fn init() Rng { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io) !Bufferized(Rng) { + pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { return .{ - ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), }; } - pub fn deinitBuffer(self: *Bufferized(Rng)) void { + pub fn deinitBuffer(self: *Buffer) void { self._state.deinit(); } @@ -499,15 +850,15 @@ pub const Tensor = struct { /// The output is guaranteed to be deterministic function of `self` Rng state, /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const op = dialects.stablehlo.rng_bit_generator( + const op = stablehlo.rng_bit_generator( mlirCtx(), self.algorithm, self._state.value(), - mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), self._state.shape()), + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; } fn update(self: Rng, new_state: *const mlir.Value) Rng { @@ -553,7 +904,8 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; + const result = floats.convert(shape_.dtype()); + return .{ rng, _resultPropagateSharding(shape_, result.value()) }; } test uniform { @@ -585,21 +937,23 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Compute stats over a uniform distribution on [-2, 10]. - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); - var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); - defer rand._state.deinit(); + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{rng_buffer}, .{ &rng_buffer, &stats }, .{ .wait = true }); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.min.deinit(); defer stats.max.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(u128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); @@ -629,8 +983,8 @@ pub const Tensor = struct { const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); - const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, op.result(0)); + const op = stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); + return _resultPropagateSharding(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -648,7 +1002,8 @@ pub const Tensor = struct { // We don't want 0 to be sampled otherwise `log` will return -inf. .{ .min = std.math.floatEps(f32), .max = 1 }, ); - return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; + const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); + return .{ rand, _resultPropagateSharding(shape_, result.value()) }; } test gumbel { @@ -689,25 +1044,28 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); + const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); defer exe.deinit(); - var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); - var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); - defer rand._state.deinit(); + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rng_buffer, &stats }, .{ .wait = true }); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.actual_dist.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(i128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); @@ -734,11 +1092,11 @@ pub const Tensor = struct { stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { + fn convolution(self: Tensor, other: Tensor, opts: stablehlo.ConvolutionOpts) Tensor { stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); stdx.debug.guard(opts.window_strides.len == N - 2, @src()); @@ -821,13 +1179,13 @@ pub const Tensor = struct { } // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = dialects.stablehlo.convolution( + const op = stablehlo.convolution( mlirCtx(), self.value(), other.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), used_opts, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); @@ -922,42 +1280,47 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise addition of the input Tensors. pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp("add", dialects.stablehlo.add)(self, other); + return binaryOp("add", stablehlo.add)(self, other); } /// Returns a Tensor containing the element-wise subtraction of the input Tensors. pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); + return binaryOp("subtract", stablehlo.subtract)(self, other); } /// Returns a Tensor containing the element-wise multiplication of the input Tensors. pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp("mul", dialects.stablehlo.multiply)(self, other); + return binaryOp("mul", stablehlo.multiply)(self, other); } /// Returns a Tensor containing the element-wise division of the input Tensors. pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp("div", dialects.stablehlo.divide)(self, other); + return binaryOp("div", stablehlo.divide)(self, other); } /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp("pow", dialects.stablehlo.power)(self, other); + return binaryOp("pow", stablehlo.power)(self, other); } /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); + return binaryOp("maximum", stablehlo.maximum)(self, other); } /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); + return binaryOp("minimum", stablehlo.minimum)(self, other); } /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. pub fn remainder(self: Tensor, other: Tensor) Tensor { - return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); + return binaryOp("remainder", stablehlo.remainder)(self, other); + } + + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn remainderConst(self: Tensor, b: anytype) Tensor { + return self.remainder(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. @@ -965,6 +1328,11 @@ pub const Tensor = struct { return self.add(.scalar(b, self.dtype())); } + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn subConstant(self: Tensor, b: anytype) Tensor { + return self.sub(.scalar(b, self.dtype())); + } + /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. pub fn divByConst(self: Tensor, b: anytype) Tensor { return self.div(.scalar(b, self.dtype())); @@ -985,20 +1353,20 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise logical operation of the input Tensors. pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { - .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), - .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), - .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), + .OR => binaryOp("or", stablehlo.or_)(self, other), + .XOR => binaryOp("xor", stablehlo.xor)(self, other), + .AND => binaryOp("and", stablehlo.and_)(self, other), }; } /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + return _result(self._shape, stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); + return _result(self._shape, stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. @@ -1007,68 +1375,144 @@ pub const Tensor = struct { return self; } - const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); - const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); + const op = stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } - test convert { + test "convert f32 -> f4e2m1" { const floats = @import("floats.zig"); const zml = @import("zml.zig"); const platform = zml.testing.env(); - - // f4e2m1 - { - const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - var x_f4: [x.len]floats.Float4E2M1 = undefined; - for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); - - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); - defer exe.deinit(); - - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); - - var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f4_xla.free(std.testing.allocator); - - errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); - try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); + const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; + for (&x_f4, 0..) |*f4_pack, i| { + f4_pack.* = .fromF32(x_f32[2 * i], x_f32[2 * i + 1]); } - // f8e3m4 - { - const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; - var x_f8e3: [x.len]floats.Float8E3M4 = undefined; - for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + const x_d: Tensor = .init(.{x_f32.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f4e2m1 }, + platform, + .{}, + ); + defer exe.deinit(); - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); - defer exe.deinit(); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); + defer x_d_buffer.deinit(); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); + var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f4_xla_d.deinit(); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); + const x_f4_items = x_f4_xla.items(floats.Float4E2M1.Packed); - var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); - const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f8e3_xla.free(std.testing.allocator); + errdefer std.log.warn("convert(f32 -> f4e2m1) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x_f32), stdx.fmt.slice(&x_f4), x_f4_xla }); + try std.testing.expectEqualSlices(floats.Float4E2M1.Packed, &x_f4, x_f4_items); + } - errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); - try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); - } + test "convert f32 -> f8e3m4" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]zml.floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f8e3m4 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f8e3_xla_d.deinit(); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); + + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x), stdx.fmt.slice(&x_f8e3), x_f8e3_xla }); + try std.testing.expectEqualSlices(u8, @ptrCast(&x_f8e3), x_f8e3_xla.constData()); + } + + test "convert u2 -> u8" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; + comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; + + const x_u2_t: Tensor = .init(.{8}, .u2); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_u2_t, .u8 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); + defer x_d_buffer.deinit(); + + var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_u8_xla_d.deinit(); + const x_u8_xla = try x_u8_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_u8_xla.free(std.testing.allocator); + + try std.testing.expectEqualSlices(u8, &x_u8, x_u8_xla.constData()); + } + + test "convert f4e2m1 -> f32" { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + + const x_t: Tensor = .init(.{x_f4_packed.len * 2}, .f4e2m1); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_t, .f32 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); + defer x_f4_d.deinit(); + + var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); + defer x_f32_d.deinit(); + const x_f32_h = try x_f32_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f32_h.free(std.testing.allocator); + + const x_f32_expected = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + errdefer std.log.warn("convert(f4e2m1x2 -> f32) failed !\ninput f4e2m1x2:\n{e}\nexpected:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x_f4_packed), x_f32_expected, x_f32_h }); + try std.testing.expectEqualSlices(f32, &x_f32_expected, x_f32_h.items(f32)); } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const round_op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1077,21 +1521,52 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); - stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); + const dot_axes = lhs.dotAxes(rhs, args); + return lhs.dotGeneral(rhs, dot_axes.contracting.constSlice(), dot_axes.batching.constSlice()); + } - const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); - const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); + pub const DotAxes = struct { + contracting: stdx.BoundedArray([2]i8, constants.MAX_RANK), + batching: stdx.BoundedArray([2]i8, constants.MAX_RANK), + }; - var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; + /// Returns contracting and batching axis pairs inferred from one or more tags. + pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { + const T = @TypeOf(tags); + if (comptime T == Shape.Tag or T == @EnumLiteral()) { + return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); + } + + const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); + const rhs_contracting_axes, _ = rhs.shape().parseAxes(tags); + var contracting_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; + for (lhs_contracting_axes.constSlice(), rhs_contracting_axes.constSlice()) |l, r| { + contracting_axes.appendAssumeCapacity(.{ @intCast(l), @intCast(r) }); + } + + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; for (0..lhs.rank()) |lhs_tag_index| { + if (std.mem.indexOfScalar(u3, lhs_contracting_axes.constSlice(), @intCast(lhs_tag_index)) != null) { + continue; + } + const lhs_tag = lhs.shape().tag(lhs_tag_index); - if (lhs_tag == Shape.toTag(args)) continue; if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); } } - return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); + + return .{ .contracting = contracting_axes, .batching = batching_axes }; + } + + test dotAxes { + const lhs: Tensor = .init(.{ .a = 20, .b = 21, .c = 22 }, .f32); + const rhs: Tensor = .init(.{ .c = 22, .b = 21, .d = 23, .a = 20 }, .f32); + + const dot_axes = lhs.dotAxes(rhs, .{ .c, .a }); + + try std.testing.expectEqualSlices([2]i8, &.{ .{ 2, 0 }, .{ 0, 3 } }, dot_axes.contracting.constSlice()); + try std.testing.expectEqualSlices([2]i8, &.{.{ 1, 1 }}, dot_axes.batching.constSlice()); } test dot { @@ -1130,7 +1605,14 @@ pub const Tensor = struct { } }.forward; - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + forward, + .{ x, y, ctr }, + platform, + .{}, + ); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); @@ -1151,10 +1633,10 @@ pub const Tensor = struct { const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + var res_shape: Shape = .scalar(lhs.dtype()); // Validate batching axes - var lhs_batching_axes: Axes = .{}; - var rhs_batching_axes: Axes = .{}; + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; for (batching_axes) |b_axes| { const l, const r = b_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1166,8 +1648,8 @@ pub const Tensor = struct { } // Validate contracting axes - var lhs_contracting_axes: Axes = .{}; - var rhs_contracting_axes: Axes = .{}; + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; for (contracting_axes) |c_axes| { const l, const r = c_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1197,11 +1679,11 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const op = dialects.stablehlo.dot_general( + const op = stablehlo.dot_general( mlirCtx(), lhs.value(), rhs.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), @@ -1209,14 +1691,14 @@ pub const Tensor = struct { .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), .dot_precision = .fast, }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1241,25 +1723,193 @@ pub const Tensor = struct { const platform = zml.testing.env(); const input: Tensor = .init(.{2}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.leakyReLU, + .{ input, 0.1 }, + platform, + .{}, + ); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); defer res.deinit(); const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); - try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { - const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); - const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, v: Tensor, b: Tensor, c: Tensor, tag: Shape.Tag) Tensor { + var value_path = self.dot(w, tag); + value_path = value_path.add(b.broad(value_path.shape())); - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); + const sigmoid_tensor = value_path.scale(beta).sigmoid(); + + var gate_path = self.dot(v, tag); + gate_path = gate_path.add(c.broad(gate_path.shape())); + + return value_path.mul(sigmoid_tensor).outer(gate_path); + } + + test swiglu { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const input: Tensor = .init(.{ .d = 4 }, .f32); + const beta = 3; + const w: Tensor = .init(.{ .c = 3, .d = 4 }, .f32); + const v: Tensor = .init(.{ .e = 3, .d = 4 }, .f32); + const b: Tensor = .init(.{ .c = 3 }, .f32); + const c: Tensor = .init(.{ .e = 3 }, .f32); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.swiglu, + .{ input, beta, w, v, b, c, Shape.toTag(.d) }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + defer input_buffer.deinit(); + + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ + .{ 1, 0, 2, 0 }, + .{ 2, 1, 3, 3 }, + .{ 1, 1, 0, 0 }, + })); + defer w_buffer.deinit(); + + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ + .{ 0, 0, 2, 1 }, + .{ 1, 2, 1, 3 }, + .{ 0, 0, 2, 0 }, + })); + defer v_buffer.deinit(); + + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + defer b_buffer.deinit(); + + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + defer c_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); + defer res.deinit(); + + const output_shape = Shape.init(.{ .c = 3, .e = 3 }, .f32); + const expectation: zml.Slice = .init(output_shape, std.mem.sliceAsBytes(&[3][3]f32{ + .{ 23.999856, 39.99976, 15.999904 }, + .{ 72.000000, 120.000000, 48.000000 }, + .{ 11.970324, 19.950540, 7.980216 }, + })); + + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); + } + + /// Returns an element-wise approximation of the error function, computed and returned in FP32. + /// Uses the Eigen rational approximation from CHLO's FP32 erf legalization: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1038 + pub fn erf(input: Tensor) Tensor { + const x = input.convert(.f32).clamp(.scalar(-4, .f32), .scalar(4, .f32)); + const x2 = x.mul(x); + // Coefficients run from the highest degree to the constant term. + const alpha = [_]f32{ + -2.72614225801306e-10, + 2.77068142495902e-08, + -2.10102402082508e-06, + -5.69250639462346e-05, + -7.34990630326855e-04, + -2.95459980854025e-03, + -1.60960333262415e-02, + }; + const beta = [_]f32{ + -1.45660718464996e-05, + -2.13374055278905e-04, + -1.68282697438203e-03, + -7.37332916720468e-03, + -1.42647390514189e-02, + }; + var numerator = x2.scale(alpha[0]).addConstant(alpha[1]); + inline for (alpha[2..]) |coefficient| { + numerator = numerator.mul(x2).addConstant(coefficient); + } + var denominator = x2.scale(beta[0]).addConstant(beta[1]); + inline for (beta[2..]) |coefficient| { + denominator = denominator.mul(x2).addConstant(coefficient); + } + return x.mul(numerator).div(denominator).clamp(.scalar(-1, .f32), .scalar(1, .f32)); + } + + /// Returns an element-wise approximation of the complementary error function in FP32. + /// Uses CHLO's Cephes-based approximation to preserve the small positive tail: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1018 + pub fn erfc(input: Tensor) Tensor { + const x = input.convert(.f32); + const abs_x = x.abs(); + const x2 = x.mul(x); + const one = Tensor.scalar(1, .f32); + + // For |x| < 1, erf(x) = x T(x^2). Coefficients run from highest degree to constant. + const t = [_]f32{ + 7.853861353153693e-5, + -8.010193625184903e-4, + 5.188327685732524e-3, + -2.685381193529856e-2, + 1.128358514861418e-1, + -3.761262582423300e-1, + 1.128379165726710, + }; + var small_poly = x2.scale(t[0]).addConstant(t[1]); + inline for (t[2..]) |coefficient| { + small_poly = small_poly.mul(x2).addConstant(coefficient); + } + const small = x.mul(small_poly).negate().addConstant(1); + + // For |x| >= 1, evaluate exp(-x^2) / |x| times P(1/x^2) or R(1/x^2). + const p = [_]f32{ + 2.326819970068386e-2, + -1.387039388740657e-1, + 3.687424674597105e-1, + -5.824733027278666e-1, + 6.210004621745983e-1, + -4.944515323274145e-1, + 3.404879937665872e-1, + -2.741127028184656e-1, + 5.638259427386472e-1, + }; + const r = [_]f32{ + -1.047766399936249e1, + 1.297719955372516e1, + -7.495518717768503, + 2.921019019210786, + -1.015265279202700, + 4.218463358204948e-1, + -2.820767439740514e-1, + 5.641895067754075e-1, + }; + const reciprocal_x2 = one.div(x2); + var middle_poly = reciprocal_x2.scale(p[0]).addConstant(p[1]); + inline for (p[2..]) |coefficient| { + middle_poly = middle_poly.mul(reciprocal_x2).addConstant(coefficient); + } + var tail_poly = reciprocal_x2.scale(r[0]).addConstant(r[1]); + inline for (r[2..]) |coefficient| { + tail_poly = tail_poly.mul(reciprocal_x2).addConstant(coefficient); + } + const poly = abs_x.cmp(.LT, .scalar(2, .f32)).select(middle_poly, tail_poly); + const z = x2.negate(); + const tail = z.exp().mul(one.div(abs_x)).mul(poly); + // Match CHLO's underflow cutoff, then use erfc(-x) = 2 - erfc(x). + const positive = z.cmp(.LT, .scalar(-88.72283905206835, .f32)).select(.scalar(0, .f32), tail); + const large = x.cmp(.LT, .scalar(0, .f32)).select(positive.negate().addConstant(2), positive); + return abs_x.cmp(.LT, one).select(small, large); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -1354,30 +2004,28 @@ pub const Tensor = struct { const rk = self.rank(); const a = self.axis(axis_); - const ones = [_]i64{1} ** constants.MAX_RANK; - var window_dimensions = ones; + var window_dimensions = constants.ones_i64; window_dimensions[a] = self.dim(a); - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a] = .{ self.dim(a) - 1, 0 }; const result = ops.reduceWindow( + 1, .{self}, - .{Tensor.scalar(0, self.dtype())}, + .{.scalar(0, self.dtype())}, .{ - .base_dilations = ones[0..rk], - .window_dilations = ones[0..rk], - .window_strides = ones[0..rk], + .base_dilations = constants.ones_i64[0..rk], + .window_dilations = constants.ones_i64[0..rk], + .window_strides = constants.ones_i64[0..rk], .window_dimensions = window_dimensions[0..rk], .padding = padding[0..rk], }, struct { - fn add(values: ops.ReduceArgs) struct { Tensor } { - return .{values.left.add(values.right)}; + fn add(values: ops.ReduceArgs) [1]Tensor { + return .{.add(values.left, values.right)}; } }.add, - .{}, ); - return result[0]; } @@ -1397,13 +2045,20 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._cumsum, + .{x}, + platform, + .{}, + ); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); @@ -1435,19 +2090,19 @@ pub const Tensor = struct { return self.reshape(res_shape); } - const op = dialects.stablehlo.transpose( + const op = stablehlo.transpose( mlirCtx(), self.value(), - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .permutation = toI64(permutation).constSlice() }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { if (self.axis(a) == self.axis(b)) return self; - var perm: Shape.AxesArray = .{}; + var perm: Shape.AxesArray = .empty; for (0..self.rank()) |i| { perm.appendAssumeCapacity(@intCast(i)); } @@ -1467,11 +2122,11 @@ pub const Tensor = struct { const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - const reshaped_val = dialects.stablehlo.reshape( + const reshaped_val = stablehlo.reshape( mlirCtx(), self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1483,11 +2138,11 @@ pub const Tensor = struct { pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { const new_shape = self._shape.splitAxis(ax, split_shape); - const reshaped_val = dialects.stablehlo.reshape( + const reshaped_val = stablehlo.reshape( mlirCtx(), self.value(), - mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), + currentLoc(), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1518,55 +2173,143 @@ pub const Tensor = struct { pub const Slice = struct { start: i64 = 0, + dyn_start: ?Tensor = null, + len: i64 = to_the_end, end: i64 = to_the_end, step: i32 = 1, singleton: bool = false, - const full = .{ .start = 0, .end = to_the_end, .step = 1 }; + /// Select the full slice + pub const full: Slice = .{ .start = 0, .end = to_the_end, .step = 1 }; + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) pub fn single(offset: i64) Slice { return .{ .start = offset, .end = offset + 1, .singleton = true }; } + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) + pub fn dyn(start: Tensor, len: i64) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dyn expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = len }; + } + + pub fn dynSingle(start: Tensor) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dynSingle expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = 1, .singleton = true }; + } + pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; - const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; + const end = if (s.singleton) + start + 1 + else if (s.end == to_the_end) + d + else if (s.end < 0) + d + s.end + else if (s.len != to_the_end) + start + s.len + else + s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; - stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); + stdx.debug.assert(end <= d, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); return res; } const to_the_end = std.math.maxInt(i64); - pub fn format(self: Slice, writer: *std.Io.Writer) !void { - if (self.singleton) { - try writer.print("[{d}]", .{self.start}); - } else if (self.end == to_the_end and self.step == 1) { - try writer.print("[{d}..]", .{self.start}); - } else if (self.step == 1) { - try writer.print("[{d}..{d}]", .{ self.start, self.end }); + pub fn format(s: Slice, writer: *std.Io.Writer) !void { + const end = if (s.len != to_the_end) s.start +| s.len else s.end; + if (s.dyn_start) |_| { + if (s.singleton) + try writer.writeAll("[x]") + else + try writer.print("[x..x+{}]", .{s.len}); + } else if (s.singleton) { + try writer.print("[{d}]", .{s.start}); + } else if (end == to_the_end and s.step == 1) { + try writer.print("[{d}..]", .{s.start}); + } else if (s.step == 1) { + try writer.print("[{d}..{d}]", .{ s.start, end }); } else { - try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); + try writer.print("[{d}..{d}:{d}]", .{ s.start, end, s.step }); } } }; - /// Slices the input Tensor over the given axis using the given parameters. - pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** constants.MAX_RANK; - slices[self.axis(axis_)] = s; - return self.slice(slices[0..self.rank()]); + /// Slice a Tensor across a specific axes. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(.a, .{ .start = 5, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(0, .single(5)); + /// ``` + pub fn slice(self: Tensor, axis_: anytype, slice_: Slice) Tensor { + return self.slices(@as([]const ResolvedAxis, &.{self.axis(axis_)}), &.{slice_}); } - /// Slices the input Tensor using the given parameters. - pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [constants.MAX_RANK]i64 = undefined; - var strides: [constants.MAX_RANK]i64 = undefined; - var limit_indices: [constants.MAX_RANK]i64 = undefined; + /// Slices a Tensor across many axes. + /// + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ .a, .b }, &.{ .start = 5, .end = 11}, .dyn(x) }); + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).slices(.{ 0, 2 }, .{.dyn(a_off, .len = 20), .{ .start = 8, .len = 12 }}); + /// ``` + pub fn slices(self: Tensor, axes_: anytype, slices_: []const Slice) Tensor { + // Note: with typed slices, the callsites are less akwards. + if (@TypeOf(axes_) != []const ResolvedAxis) { + const resolved_axes: []const ResolvedAxis = self._shape.parseAxes(axes_)[0].slice(); + return self.slices(resolved_axes, slices_); + } + + const rk = self.rank(); + any_dyn: { + for (slices_) |s| { + if (s.dyn_start) |_| break :any_dyn; + } + return self.staticSlices(axes_, slices_); + } + + const zero = scalar(0, .i64).value(); + var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(zero); + var new_shape: Shape = self._shape; + var to_remove: Shape.AxesArray = .empty; + + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step == 1, "Tensor.slices doesn't support mixing strided and dynamic slices, got {f}", .{stdx.fmt.slice(slices_)}); + new_shape._dims.buffer[a] = s.len; + start_indices[a] = if (s.dyn_start) |dyn| + dyn.convert(.i64).value() + else + scalar(s.start, .i64).value(); + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + + const op = stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..rk], + currentLoc(), + ).appendTo(currentBlock()); + + const res = _result(new_shape, op.result(0)); + return res.reshape(new_shape.removeMany(to_remove.constSlice())); + } + + fn staticSlices(self: Tensor, axes_: []const ResolvedAxis, slices_: []const Slice) Tensor { + var start_indices: [constants.MAX_RANK]i64 = @splat(0); + var strides: [constants.MAX_RANK]i64 = @splat(1); + var limit_indices: [constants.MAX_RANK]i64 = self._shape._dims.buffer; var res_shape: Shape = self._shape; - for (slices, 0..) |s, a| { - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); const args: Slice = s.absolute(self.dim(a)); start_indices[a] = args.start; @@ -1575,21 +2318,21 @@ pub const Tensor = struct { res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } - const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); - const slice_op = dialects.stablehlo.slice( - mlirCtx(), + const ctx = Compiler.current(); + const slice_op = stablehlo.slice( + ctx.mlir_ctx, self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], - result_type, - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(ctx.mlir_ctx, res_shape), + ctx.location, ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); - var to_remove: Shape.AxesArray = .{}; - for (slices, 0..) |s, a| { - if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + var to_remove: Shape.AxesArray = .empty; + for (axes_, slices_) |a, s| { + if (s.singleton) to_remove.appendAssumeCapacity(a); } return res.reshape(res_shape.removeMany(to_remove.constSlice())); } @@ -1603,10 +2346,11 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - // Wrap slice1d to hide the anytype in the signature. + // Wrap slice to hide the anytype in the signature. + // TODO(wzk): stdx.meta.specifyFn (waiting 0.17 for SoA meta programming) const Local = struct { - pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - return input.slice1d(ax, slice_); + fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + return input.slice(ax, slice_); } }; @@ -1616,10 +2360,50 @@ pub const Tensor = struct { .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._slice1dAxis, + .{ x, ax, slice_ }, + platform, + .{}, + ); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{} }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); + } + } + + test "dynamic slices" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn dynSliceResolved(x: Tensor, ax: ResolvedAxis, dyn_slice: Slice) Tensor { + return x.slice(ax, dyn_slice); + } + }; + + inline for (.{ + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, + }) |testcase| { + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; + const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); + defer z_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -1630,25 +2414,11 @@ pub const Tensor = struct { return if (idx < 0) self.dim(axis_) + idx else idx; } - pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { - return self.slice1d(axis_, .single(i)); - } - - pub fn choose(self: Tensor, offsets: anytype) Tensor { - const off, const tags = Shape.parseDimensions(offsets); - var slices = [_]Slice{.{}} ** constants.MAX_RANK; - for (off.constSlice(), tags.constSlice()) |o, t| { - const ax = self.axis(t); - slices[ax] = .single(o); - } - return self.slice(slices[0..self.rank()]); - } - /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; - stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - var buffer: [32]*const mlir.Value = undefined; + const ctx = Compiler.current(); + var buffer = ctx.alloc(*const mlir.Value, tensors.len); std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1661,7 +2431,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -1672,9 +2442,6 @@ pub const Tensor = struct { /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { - // Note: we could ask the compilation context for some memory instead of stack allocating - stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); - const shape0 = tensors[0]._shape; const res_shape = shape0.insertTag(axis_, 1, tag); @@ -1682,7 +2449,8 @@ pub const Tensor = struct { stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - var reshaped: [32]Tensor = undefined; + const ctx = Compiler.current(); + var reshaped = ctx.alloc(Tensor, tensors.len); for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); } @@ -1694,7 +2462,7 @@ pub const Tensor = struct { else shape0.axis(axis_); - return Tensor.concatenate(reshaped[0..tensors.len], ax); + return Tensor.concatenate(reshaped, ax); } /// Repeats a Tensor several times along the given axis. @@ -1746,13 +2514,21 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.repeat1d, + .{ input, ax, reps }, + platform, + .{}, + ); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -1761,17 +2537,18 @@ pub const Tensor = struct { /// /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] /// This is equivalent to repeat(ax+1) unless ax is the last axis. - pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u32) Tensor { + stdx.debug.assert(n_rep > 0, "stutter1d expects a positive repeat count, got {}", .{n_rep}); const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); + const res_shape = self._shape.setDim(a, self.dim(a) * @as(i64, n_rep)); const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. - pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { + pub fn stutter(self: Tensor, n_reps: []const u32) Tensor { // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); @@ -1801,68 +2578,76 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.stutter1d, + .{ input, ax, reps }, + platform, + .{}, + ); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const negate_op = stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const cosine_op = stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const sine_op = stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const sqrt_op = stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const rsqrt_op = stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const tanh_op = stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const expm1_op = stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } @@ -1871,7 +2656,7 @@ pub const Tensor = struct { end: i64, step: i64 = 1, }; - /// + /// Returns a Tensor containing evenly spaced values within a given interval. pub fn arange(args: ArangeArgs, dt: DataType) Tensor { stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); @@ -1879,11 +2664,11 @@ pub const Tensor = struct { const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = dialects.stablehlo.iota( + var op = stablehlo.iota( mlirCtx(), 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); @@ -1898,6 +2683,31 @@ pub const Tensor = struct { return res; } + test arange { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.arange, + .{ .{ .start = 2, .end = 26, .step = 4 }, .u32 }, + platform, + .{}, + ); + defer exe.deinit(); + + var output = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer output.deinit(); + + try zml.testing.expectClose( + std.testing.io, + zml.Slice.initConst(.init(.{6}, .u32), @ptrCast(&[_]u32{ 2, 6, 10, 14, 18, 22 })), + output, + .exact_match, + ); + } + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. /// /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. @@ -1908,13 +2718,13 @@ pub const Tensor = struct { const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); - var op = dialects.stablehlo.iota( + var op = stablehlo.iota( mlirCtx(), a, - mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), res_shape), + currentLoc(), ).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); + return _resultPropagateSharding(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -1930,11 +2740,11 @@ pub const Tensor = struct { stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); const sh = Shape.init(.{args.steps}, dt); - var iota_op = dialects.stablehlo.iota( + var iota_op = stablehlo.iota( mlirCtx(), 0, - mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), - .unknown(mlirCtx()), + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); @@ -1972,32 +2782,48 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{}); defer exe.deinit(); } /// Returns a constant Tensor with the given value. pub fn constant(val: DataType.Value) Tensor { - const op = dialects.stablehlo.constant( + const op = stablehlo.constant( mlirCtx(), &.{}, mlirx.Type.fromDType(mlirCtx(), val.dtype()), val.asBytes(), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(.init(&.{}, val.dtype()), op.result(0)); } pub fn zeroes(sh: Shape) Tensor { - return Tensor.constant(sh.dtype().zero()).broad(sh); + const result = Tensor.constant(sh.dtype().zero()).broad(sh); + return _resultPropagateSharding(sh, result.value()); + } + + pub fn uninitialized(sh: Shape) Tensor { + const ctx = Compiler.current(); + const buffer_type = mlir.Type.memRef( + mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), + sh.dims(), + null, + null, + ); + const buffer = stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) + .appendTo(currentBlock()); + const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) + .appendTo(currentBlock()); + return _resultPropagateSharding(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(sh, constant_op.result(0)); + const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); + return _resultPropagateSharding(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2031,11 +2857,8 @@ pub const Tensor = struct { // but we propagate output_shape tags. return _result(res_shape, self.value()); } - const result_type = mlir.rankedTensorType( - res_shape.dims(), - mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), - ); - const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); + const broadcast_op = stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2071,6 +2894,8 @@ pub const Tensor = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return self; + if (ops.LoweringCompatibility.preserveIntegerScalarBroadcast(self, other)) |rewritten| return rewritten; + // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -2079,18 +2904,31 @@ pub const Tensor = struct { } // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } return self.broadcast(other, axes_.constSlice()); } + pub fn optimizationBarrier(self: Tensor) Tensor { + const ctx = Compiler.current(); + + const op = stablehlo.optimizationBarrier( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + ctx.location, + ).appendTo(ctx.currentScope().block); + + return _result(self.shape(), op.result(0)); + } + /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); - const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); + const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); + const reshape_value = stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } @@ -2111,7 +2949,7 @@ pub const Tensor = struct { pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - const ZEROS = [_]i64{0} ** constants.MAX_RANK; + const ZEROS: [constants.MAX_RANK]i64 = @splat(0); var low = ZEROS; var high = ZEROS; var interior = ZEROS; @@ -2128,12 +2966,12 @@ pub const Tensor = struct { } const rk = self.rank(); - const pad_op = dialects.stablehlo.pad( + const pad_op = stablehlo.pad( mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, pad_op.result(0)); @@ -2149,9 +2987,9 @@ pub const Tensor = struct { self.axis(axis_); var res_shape = self._shape; - const ones = [_]i64{1} ** constants.MAX_RANK; + const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + res_shape._tags.insertSlice(ax, tags_) catch unreachable; return self.reshape(res_shape); } @@ -2178,7 +3016,7 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); + const reverse_op = stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } @@ -2215,7 +3053,7 @@ pub const Tensor = struct { /// so that gather can pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); - var idx_axes: Shape.AxesArray = .{}; + var idx_axes: Shape.AxesArray = .empty; for (idx_tags.slice()) |t| { idx_axes.appendAssumeCapacity(self.axis(t)); } @@ -2238,14 +3076,14 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, @@ -2329,10 +3167,10 @@ pub const Tensor = struct { // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; - var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -2359,7 +3197,7 @@ pub const Tensor = struct { } } - const gather_op = dialects.stablehlo.gather( + const gather_op = stablehlo.gather( mlirCtx(), self.value(), indices.value(), @@ -2373,7 +3211,7 @@ pub const Tensor = struct { .index_vector_dim = index_coord_axis, .indices_are_sorted = opts.indices_are_sorted, }, - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(res_shape, gather_op.result(0)); } @@ -2390,14 +3228,14 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, @@ -2433,7 +3271,14 @@ pub const Tensor = struct { // Test with actual values. const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._gatherSlices, + .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, + platform, + .{}, + ); defer exe.deinit(); var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); @@ -2444,14 +3289,14 @@ pub const Tensor = struct { return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); } }.forward; - var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{}); defer temp_exe.deinit(); - const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); + const buffer = try temp_exe.eval(std.testing.allocator, std.testing.io, {}); break :b buffer; }; defer operand_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, indices_buffer }); defer result.deinit(); const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ @@ -2464,7 +3309,7 @@ pub const Tensor = struct { .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, 0); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } pub const ScatterOpts = struct { @@ -2595,7 +3440,7 @@ pub const Tensor = struct { pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { return self.scatterSlices( - .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, + .{ .c = coords.slice(.coord, .single(0)), .b = coords.slice(.coord, .single(1)) }, updates, .{ .update_fn = ScatterOpts.increment }, ); @@ -2608,15 +3453,15 @@ pub const Tensor = struct { { // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); const idx = Local._idx; @@ -2651,17 +3496,16 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; @@ -2675,7 +3519,7 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); @@ -2684,8 +3528,7 @@ pub const Tensor = struct { defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; @@ -2698,7 +3541,7 @@ pub const Tensor = struct { const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); defer exe.deinit(); var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); @@ -2716,8 +3559,7 @@ pub const Tensor = struct { defer start_indices_buffer.deinit(); var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); - - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); const expected = [2][3][4][2]u16{ @@ -2773,18 +3615,18 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) [2]Tensor { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); // If values.left == values.right: keep the smallest idx. const is_same = values.left.cmp(.EQ, values.right); const is_first = indices.left.cmp(.LT, indices.right); const is_same_but_first = is_same.logical(.AND, is_first); const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); const max_idx = keep_left_idx.select(indices.left, indices.right); return .{ max_val, max_idx }; @@ -2816,14 +3658,17 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 1, 5 }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{}); defer exe.deinit(); { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); defer res.values.deinit(); defer res.indices.deinit(); const max_ = res.values.getValue(f32, std.testing.io); @@ -2837,7 +3682,10 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); defer res.values.deinit(); defer res.indices.deinit(); const max_ = try res.values.getValue(f32, std.testing.io); @@ -2855,9 +3703,9 @@ pub const Tensor = struct { pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; + const direction: stablehlo.ComparisonDirectionAttr.Direction = if (opts.descending) .GT else .LT; const res = ops.sort(.{ self, indices }, a, struct { - fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: stablehlo.ComparisonDirectionAttr.Direction) Tensor { return values.left.cmp(direction_, values.right); } }.call, .{direction}, true); @@ -2885,13 +3733,13 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -2904,7 +3752,7 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 1, 5, 10 }, .f16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ @@ -2916,7 +3764,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -2935,7 +3783,7 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ @@ -2950,7 +3798,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -2978,14 +3826,58 @@ pub const Tensor = struct { const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); - break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; + stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); + break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; }, else => stdx.debug.compileError(err_msg, .{}), }; - var result = self.sort(a, .{ .descending = opts.descending }); - result.values = result.values.slice1d(a, .{ .end = k }); - result.indices = result.indices.slice1d(a, .{ .end = k }); + const ctx = Compiler.current(); + var result: SortRes = switch (ctx.platform.target) { + // Work around https://github.com/aws-neuron/aws-neuron-sdk/issues/1339 until Neuron's sort+slice rewrite uses the slice size as k. + .neuron => blk: { + stdx.debug.assert(self.dtype().isFloat(), "Neuron Tensor.topK only supports float tensors, got {}", .{self.dtype()}); + stdx.debug.assert(k <= 16384, "Neuron AwsNeuronTopK supports k <= 16384, got {}", .{k}); + + const last_axis: u4 = self.rank() - 1; + const topk_input = if (a == last_axis) self else self.swapAxes(a, last_axis); + const call_input = if (opts.descending) topk_input else topk_input.negate(); + const values_shape = topk_input.shape().setDim(last_axis, @intCast(k)); + const indices_shape = values_shape.withDtype(.i32); + const result_types = [_]*const mlir.Type{ + mlirx.Type.rankedTensor(ctx.mlir_ctx, values_shape), + mlirx.Type.rankedTensor(ctx.mlir_ctx, indices_shape), + }; + + var backend_config_buf: [32]u8 = undefined; + const backend_config = std.fmt.bufPrint(&backend_config_buf, "{d}", .{k}) catch unreachable; + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{call_input.value()}, + &result_types, + .{ + .call_target_name = "AwsNeuronTopK", + .has_side_effect = false, + .backend_config = .{ .original = backend_config }, + }, + ctx.location, + ).appendTo(currentBlock()); + + var values = Tensor._result(values_shape, op.result(0)); + if (!opts.descending) values = values.negate(); + var indices = Tensor._result(indices_shape, op.result(1)); + if (a != last_axis) { + values = values.swapAxes(a, last_axis); + indices = indices.swapAxes(a, last_axis); + } + break :blk .{ .values = values, .indices = indices }; + }, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => blk: { + var sorted = self.sort(a, .{ .descending = opts.descending }); + sorted.values = sorted.values.slice(a, .{ .end = k }); + sorted.indices = sorted.indices.slice(a, .{ .end = k }); + break :blk sorted; + }, + }; if (has_name) |new_name| { result.values._shape._tags.set(a, new_name.ptr); result.indices._shape._tags.set(a, new_name.ptr); @@ -3014,7 +3906,7 @@ pub const Tensor = struct { // TODO: support maxPool on non last axis const a = self.axis(-1); - const ones = [_]i64{1} ** constants.MAX_RANK; + const ones: [constants.MAX_RANK]i64 = @splat(1); var window_dimensions = ones; window_dimensions[a] = opts.window_dimensions; var window_strides = window_dimensions; @@ -3025,12 +3917,13 @@ pub const Tensor = struct { var window_dilations = ones; window_dilations[a] = opts.window_dilations; - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a] = opts.padding; - const result = ops.reduceWindow( + const values, const indices = ops.reduceWindow( + 2, .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3038,10 +3931,9 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - MaxPoolRes.cmp, - .{}, + &MaxPoolRes.cmp, ); - return .{ .values = result[0], .indices = result[1] }; + return .{ .values = values, .indices = indices }; } /// Computes the 2d maxPool operation on the input Tensor. @@ -3064,13 +3956,14 @@ pub const Tensor = struct { const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - const result = ops.reduceWindow( + const values, const indices = ops.reduceWindow( + 2, .{ self, iota(self._shape, a) }, - .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3078,10 +3971,9 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - MaxPoolRes.cmp, - .{}, + &MaxPoolRes.cmp, ); - return .{ .values = result[0], .indices = result[1] }; + return .{ .values = values, .indices = indices }; } /// Chunk a given tensor into exactly n parts of equal shape. @@ -3093,7 +3985,7 @@ pub const Tensor = struct { var chunks: [n_chunks]Tensor = undefined; for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); + chunks[i] = self.slice(a, .{ .start = start, .end = start + chunk_size }); } return chunks; } @@ -3103,15 +3995,15 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, @@ -3134,29 +4026,21 @@ pub const Tensor = struct { pub fn chunkAllowTrailing( self: Tensor, axis_: i64, - n_chunks: comptime_int, - ) ![]Tensor { + n_chunks: usize, + ) []Tensor { const a = self.axis(axis_); - const d = self.dim(a); - const chunk_size: i64 = @divFloor(d, n_chunks); + const d: i64 = self.dim(a); + const chunk_size: i64 = @divFloor(d, @as(i64, @intCast(n_chunks))); const tail_chunk_size: i64 = @rem(d, chunk_size); - const allocator = CompilationContext.current().arena.allocator(); + const len: usize = if (tail_chunk_size == 0) n_chunks else n_chunks + 1; + const chunks = Compiler.current().alloc(Tensor, len); - var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); - defer chunks.deinit(allocator); - - for (0..n_chunks) |i| { + for (0.., chunks) |i, *chunk| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks.appendAssumeCapacity( - self.slice1d(a, .{ .start = start, .end = start + chunk_size }), - ); + chunk.* = self.slice(a, .{ .start = start, .end = @min(start + chunk_size, d) }); } - if (tail_chunk_size != 0) { - const start: i64 = n_chunks * chunk_size; - chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); - } - return chunks.toOwnedSlice(allocator); + return chunks; } test chunkAllowTrailing { @@ -3164,15 +4048,15 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - comp.pushBlock(block); - defer comp.popBlock(); + const scope = comp.pushBlock(block); + defer scope.pop(); inline for (.{ .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, @@ -3181,8 +4065,8 @@ pub const Tensor = struct { .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); - const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); + const x = Tensor.zeroes(.init(x_shape, .f16)); + const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); for (chunks[0..n_chunks]) |chk| { @@ -3207,119 +4091,51 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); - defer arena.deinit(); - - const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); + const res = Compiler.current().alloc(Tensor, split_sizes.len); var start: i64 = 0; for (split_sizes, 0..) |n, i| { - res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); + res[i] = self.slice(a, .{ .start = start, .end = start + n }); start += n; } return res; } - /// Slices the input Tensor along a specific axis, with a start offset known at runtime. - /// Note: this doesn't support tagging, if you have tags, - /// you should use `dynamicSlice` directly. - pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { - stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); - - const a = self.axis(axis_); - const new_shape = self._shape.set(a, slice_.len); - - var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; - start_indices[a] = slice_.start.value(); - - const op = dialects.stablehlo.dynamic_slice( - mlirCtx(), - self.value(), - new_shape.dims(), - start_indices[0..self.rank()], - .unknown(mlirCtx()), - ).appendTo(currentBlock()); - - return _result(new_shape, op.result(0)); - } - - pub const DynSlice = struct { start: Tensor, len: i64 }; - - /// Slices a Tensor across many axes, with runtime known offsets. - /// - /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. - /// When using the tagged API it is allowed to not specify some axes. - /// But with the non-tagged API all slices need to be specified. - /// Examples: - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ - /// .a = .{ .start = a_off, .len = 11 }, - /// .b = .{ .start = b_off, .len = 12 }, - /// }); - /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); - /// ``` - pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { - // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? - // - const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); - - // TODO use slices and slices_tags for the format. - // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" - // which is kinda ugly. - - const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; - const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; - var res_shape = self._shape; - for (slices.constSlice(), 0..) |slice_, i| { - const offset = slice_.start; - const len = slice_.len; - if (slices_tags.len == 0) { - stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); - - offset_values[i] = offset.value(); - res_shape._dims.set(i, len); - - stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); - } else { - const t = slices_tags.get(i); - const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); - - stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); - - offset_values[a] = offset.value(); - res_shape._dims.set(a, len); - } - } - const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); - return _result(res_shape, op.result(0)); - } - - test dynamicSlice { + test split { const zml = @import("zml.zig"); const platform = zml.testing.env(); - inline for (.{ - .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, - .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, - }) |testcase| { - const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; - const x: Tensor = .fromShape(x_shape); - const z: Tensor = .init(.{}, .i32); + const Local = struct { + pub fn _split(x: Tensor, axis_: i8, split_sizes: []const i64) []Tensor { + return x.split(axis_, split_sizes); + } + }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); + inline for (.{ + .{ [3]f32{ 0, 1, 2 }, Shape.init(.{3}, .f32), 0, [3]i64{ 1, 1, 1 }, &.{ [1]f32{0}, [1]f32{1}, [1]f32{2} } }, + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), 0, [2]i64{ 1, 9 }, &.{ [1]f32{0}, [9]f32{ 1, 2, 3, 4, 5, 6, 7, 8, 9 } } }, + .{ [4][3]f32{ .{ 0, 1, 2 }, .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } }, Shape.init(.{ 4, 3 }, .f32), 0, [2]i64{ 1, 3 }, &.{ [1][3]f32{.{ 0, 1, 2 }}, [3][3]f32{ .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } } } }, + .{ [2][6]f32{ .{ 0, 1, 2, 3, 4, 5 }, .{ 6, 7, 8, 9, 10, 11 } }, Shape.init(.{ 2, 6 }, .f32), 1, [2]i64{ 2, 4 }, &.{ [2][2]f32{ .{ 0, 1 }, .{ 6, 7 } }, [2][4]f32{ .{ 2, 3, 4, 5 }, .{ 8, 9, 10, 11 } } } }, + }) |testcase| { + const x_data, const x_shape, const ax, const args, const expectation = testcase; + const x: Tensor = .fromShape(x_shape); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); - defer z_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); - defer res.deinit(); + const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); + defer std.testing.allocator.free(res); + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, res, .{ .wait = true }); + defer for (res) |*buffer| buffer.deinit(); - try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); + inline for (expectation, 0..) |expected, i| { + try std.testing.expectEqual(expected, try res[i].getValue(@TypeOf(expected), std.testing.io)); + } } } @@ -3327,7 +4143,7 @@ pub const Tensor = struct { /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; + var start_indices: [constants.MAX_RANK]Tensor = @splat(placeholder); start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -3402,19 +4218,19 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = .{zero} ** constants.MAX_RANK; + offset_values = @splat(zero); for (offset.constSlice(), offset_tags.constSlice()) |start, t| { const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); } } - const op = dialects.stablehlo.dynamic_update_slice( + const op = stablehlo.dynamic_update_slice( mlirCtx(), self.value(), update.value(), offset_values[0..self.rank()], - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -3432,7 +4248,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); @@ -3442,7 +4258,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); @@ -3458,7 +4274,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -3468,7 +4284,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3487,7 +4303,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -3497,7 +4313,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3517,7 +4333,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -3529,7 +4345,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3549,7 +4365,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(&idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -3561,7 +4377,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -3572,7 +4388,7 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { + pub fn cmp(self: Tensor, direction: stablehlo.ComparisonDirectionAttr.Direction, other: Tensor) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); @@ -3580,13 +4396,13 @@ pub const Tensor = struct { stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - const op = dialects.stablehlo.compare( + const op = stablehlo.compare( mlirCtx(), self.value(), other.value(), - dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), - getComparisonType(mlirCtx(), self.dtype()).getValue(), - .unknown(mlirCtx()), + stablehlo.ComparisonDirectionAttr.get(mlirCtx(), .{ .value = direction }).value(), + getComparisonType(mlirCtx(), self.dtype()).value(), + currentLoc(), ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); @@ -3598,12 +4414,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); - const p = self.shape()._partitioning.get(a); + const p = self.shape()._sharding.partition.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - // TODO(Corentin): Not sure about that - res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; + + res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); @@ -3629,23 +4445,20 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 2, 2 }, .u8); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); + try std.testing.expectEqual( - [2][2][2]u8{ .{ - .{ 1, 0 }, - .{ 0, 2 }, - }, .{ - .{ 3, 0 }, - .{ 0, 4 }, - } }, + [2][2][2]u8{ + .{ .{ 1, 0 }, .{ 0, 2 } }, + .{ .{ 3, 0 }, .{ 0, 4 } }, + }, try res.getValue([2][2][2]u8, std.testing.io), ); } @@ -3691,10 +4504,10 @@ pub const Tensor = struct { defer x_buffer.deinit(); { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -3707,10 +4520,10 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -3723,10 +4536,10 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{}); defer exe.deinit(); - var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -3756,20 +4569,24 @@ pub const Tensor = struct { stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); - const op = dialects.stablehlo.select( + const op = stablehlo.select( mlirCtx(), bool_tensor.value(), on_true.value(), on_false.value(), - .unknown(mlirCtx()), + currentLoc(), ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); } + pub fn mask(x: Tensor, pred: Tensor, mask_value: anytype) Tensor { + return pred.broad(x.shape()).select(x, .scalar(mask_value, x.dtype())); + } + /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); + const op = stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -3858,7 +4675,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); @@ -3866,7 +4683,11 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); + var xs: zml.Buffer = undefined; + var ys: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ x_buffer, y_buffer }, .{ &xs, &ys }, .{ .wait = true }); defer xs.deinit(); defer ys.deinit(); @@ -3902,8 +4723,8 @@ pub const Tensor = struct { const k = @mod(shift, n); // normalize shift to [0, n) if (k == 0) return self; return Tensor.concatenate(&.{ - self.slice1d(a, .{ .start = n - k, .end = n }), - self.slice1d(a, .{ .start = 0, .end = n - k }), + self.slice(a, .{ .start = n - k, .end = n }), + self.slice(a, .{ .start = 0, .end = n - k }), }, a); } @@ -3913,7 +4734,7 @@ pub const Tensor = struct { /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; + var out: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{ .buffer = undefined, .len = vectors.len }; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -3931,7 +4752,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); @@ -3939,7 +4760,7 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); + var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); defer z.deinit(); try std.testing.expectEqualDeep( @@ -3993,31 +4814,40 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - _ = ops.customCall("zml$print", .{input}, .{}, .{ .name = name }, .{ .has_side_effect = true }); + const ctx = Compiler.current(); + const full_name = std.fmt.allocPrint(ctx.arena.allocator(), "{s}: {f}", .{ name, input.shape() }) catch @panic("OOM"); + defer ctx.arena.allocator().free(full_name); + switch (ctx.platform.target) { + .cpu, .cuda, .rocm, .tpu, .metal => { + ops.manualComputation((struct { + input: Tensor, + name: []const u8, + + fn body(body_ctx: @This(), _: void) void { + ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); + } + }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); + }, + .oneapi, .neuron, .furiosa => {}, + } } fn mlirCtx() *mlir.Context { - return CompilationContext.current().mlir_ctx; + return Compiler.current().mlir_ctx; + } + + fn currentLoc() *const mlir.Location { + return Compiler.current().location; } fn currentBlock() *mlir.Block { - return CompilationContext.current().currentScope().block; - } - - /// Returns the donation data of the tensor. - pub fn donation(self: Tensor) ?usize { - return CompilationContext.current().currentScope().id_to_donation.get(self.id); - } - - /// Returns the output memory kind of the tensor. - pub fn outputMemoryKind(self: Tensor) Memory.Kind { - return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; + return Compiler.current().currentScope().block; } }; fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result = [_]i64{1} ** constants.MAX_RANK; + var result: [constants.MAX_RANK]i64 = @splat(1); const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; @@ -4025,7 +4855,7 @@ fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims = [_]i64{1} ** constants.MAX_RANK; + var input_dims: [constants.MAX_RANK]i64 = @splat(1); @memcpy(input_dims[0..in_dims.len], in_dims); const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; @@ -4047,13 +4877,13 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(co return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { - return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.CompareTypeAttr { + return .get(ctx, .{ .value = switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, .float => .FLOAT, .complex => @panic("Can't compare complex numbers"), - }); + } }); } test "Tensor.maxPool1d" { @@ -4074,13 +4904,16 @@ test "Tensor.maxPool1d" { const x: Tensor = .init(.{ 2, 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); defer result.values.deinit(); defer result.indices.deinit(); @@ -4113,13 +4946,16 @@ test "Tensor.maxPool2d" { const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); defer result.values.deinit(); defer result.indices.deinit(); @@ -4155,13 +4991,13 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr } fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .empty; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { @@ -4213,6 +5049,13 @@ test "unused tensor" { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{10}, .f32)}, + platform, + .{}, + ); defer exe.deinit(); } diff --git a/zml/test_runner.zig b/zml/test_runner.zig deleted file mode 100644 index 121b9be4..00000000 --- a/zml/test_runner.zig +++ /dev/null @@ -1,112 +0,0 @@ -//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -// note: std_options.log_level does not respect testing.log_level -// ref: https://github.com/ziglang/zig/issues/5738 -const log_level: std.log.Level = .warn; - -pub const std_options: std.Options = .{ - .log_level = log_level, -}; - -var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; -var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); - -pub fn main() !void { - testing.log_level = log_level; - - const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; - var ok_count: usize = 0; - var skip_count: usize = 0; - var fail_count: usize = 0; - const root_node = std.Progress.start(std.testing.io, .{ - .root_name = "Test", - .estimated_total_items = test_fn_list.len, - }); - - const have_tty = try std.Io.File.stderr().isTty(std.testing.io); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const identifier_query = if (args.next()) |arg| blk: { - std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); - break :blk arg; - } else blk: { - break :blk ""; - }; - - var leaks: usize = 0; - - for (test_fn_list, 0..) |test_fn, i| { - if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { - continue; - } - - testing.allocator_instance = .{}; - defer { - if (testing.allocator_instance.deinit() == .leak) { - leaks += 1; - } - } - - testing.io_instance = .init(testing.allocator_instance.allocator(), .{}); - defer testing.io_instance.deinit(); - - const test_node = root_node.start(test_fn.name, 0); - if (!have_tty) { - std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); - } - if (test_fn.func()) |_| { - ok_count += 1; - test_node.end(); - if (!have_tty) std.debug.print("OK\n", .{}); - } else |err| switch (err) { - error.SkipZigTest => { - skip_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); - } else { - std.debug.print("SKIP\n", .{}); - } - test_node.end(); - }, - else => { - fail_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ - i + 1, test_fn_list.len, test_fn.name, @errorName(err), - }); - } else { - std.debug.print("FAIL ({s})\n", .{@errorName(err)}); - } - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace); - } - test_node.end(); - }, - } - } - root_node.end(); - if (ok_count == test_fn_list.len) { - std.debug.print("All {d} tests passed.\n", .{ok_count}); - } else { - std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); - } - if (log_err_count != 0) { - std.debug.print("{d} errors were logged.\n", .{log_err_count}); - } - if (leaks != 0) { - std.debug.print("{d} tests leaked memory.\n", .{leaks}); - } - if (leaks != 0 or log_err_count != 0 or fail_count != 0) { - std.process.exit(1); - } - - // Explicit exit 0 to exit from async main thread and main properly - std.process.exit(0); -} diff --git a/zml/testing.zig b/zml/testing.zig index 8e33ed58..5e3d6333 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -3,8 +3,9 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const Slice = @import("slice.zig").Slice; const Platform = @import("platform.zig").Platform; +const Sharding = @import("Sharding.zig"); +const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); @@ -14,14 +15,32 @@ var _platform: ?*const Platform = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - _platform = Platform.auto(std.heap.c_allocator, std.testing.io, .{ - .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, - }) catch unreachable; + const memory_fraction = 0.90; + const platform = Platform.auto( + std.heap.c_allocator, + std.testing.io, + .{ + .cuda = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .rocm = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .metal = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, + .oneapi = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, + }, + ) catch @panic("Pjrt not available"); + + // Register meshes required by flash attention. + _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + _platform = platform; } return _platform.?; } +pub fn tpMesh() *const Sharding.Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.testing.tpMesh() outside of a test block"); + + return _platform.?.meshes.get("model").?; +} + /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. @@ -79,7 +98,6 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e8m0, => |t| { const L = t.toZigType(); - const left_data = left.constItems(L); switch (right.dtype()) { inline .bf16, .f16, @@ -92,22 +110,24 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e5m2fnuz, => |rt| { const R = rt.toZigType(); - const right_data = right.constItems(R); - const compare_report = try compareSlices(allocator, L, R, left_data, right_data, opts); - const ok = !compare_report.nan_or_inf and compare_report.close_fraction >= opts.minimum_close_fraction; - if (ok) { - return; - } else { - try w.print("{f}\n", .{compare_report}); + const report: CompareOpts.Report = try .compareSlices(true, allocator, L, R, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); return error.TestUnexpectedResult; } + return; }, else => unreachable, } }, inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - return std.testing.expectEqualSlices(T, left.constItems(T), right.constItems(T)); + const report: CompareOpts.Report = try .compareSlices(false, allocator, T, T, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); + return error.TestUnexpectedResult; + } + return; }, .c64, .c128 => @panic("TODO: support comparison of complex"), } @@ -118,113 +138,224 @@ pub const CompareOpts = struct { relative_tolerance: f32 = 1e-2, epsilon_relative: f32 = 1e-6, minimum_close_fraction: f32 = 0.999, -}; -pub const CompareReport = struct { - nan_or_inf: bool, - max_absolute_error: f32, - mean_absolute_error: f32, - rmse: f32, - - close_fraction: f32, - - p50_absolute_error: f32, - p90_absolute_error: f32, - p99_absolute_error: f32, - p999_absolute_error: f32, - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print( - \\ nan_or_inf: {} - \\ max_absolute_error: {d} - \\ mean_absolute_error: {d} - \\ rmse: {d} - \\ close_fraction: {d} - \\ p50_absolute_error: {d} - \\ p90_absolute_error: {d} - \\ p99_absolute_error: {d} - \\ p999_absolute_error: {d} - , .{ - self.nan_or_inf, - self.max_absolute_error, - self.mean_absolute_error, - self.rmse, - self.close_fraction, - self.p50_absolute_error, - self.p90_absolute_error, - self.p99_absolute_error, - self.p999_absolute_error, - }); - } -}; - -pub fn compareSlices(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareReport { - var nan_or_inf: bool = false; - var max_absolute_error: f32 = 0; - var sum_absolute_error: f64 = 0; - var sum_squared_error: f64 = 0; - var count_close: usize = 0; - var absolute_errors = try allocator.alloc(f32, left.len); - defer allocator.free(absolute_errors); - var relative_errors = try allocator.alloc(f32, left.len); - defer allocator.free(relative_errors); - - for (left, right, 0..) |l, r, i| { - const l_f32 = zml.floats.floatCast(f32, l); - const r_f32 = zml.floats.floatCast(f32, r); - if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { - nan_or_inf = true; - continue; - } - const absolute_error = @abs(l_f32 - r_f32); - max_absolute_error = @max(max_absolute_error, absolute_error); - sum_absolute_error += @as(f64, @floatCast(absolute_error)); - sum_squared_error += @as(f64, @floatCast(absolute_error)) * @as(f64, @floatCast(absolute_error)); - - const scale = @max(@abs(l_f32), @abs(r_f32)); - const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; - if (absolute_error <= tolerance) { - count_close += 1; - } - - absolute_errors[i] = absolute_error; - const denom = @max(opts.epsilon_relative, scale); - relative_errors[i] = absolute_error / denom; - } - - std.sort.heap(f32, absolute_errors, {}, std.sort.asc(f32)); - std.sort.heap(f32, relative_errors, {}, std.sort.asc(f32)); - - const q = struct { - pub fn q(values: []const f32, frac: f32) f32 { - if (values.len == 0) return 0; - const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); - return values[idx]; - } - }.q; - - const mean_absolute_error: f32 = @floatCast(sum_absolute_error / @as(f64, @floatFromInt(left.len))); - const rmse: f32 = @floatCast(std.math.sqrt(sum_squared_error / @as(f64, @floatFromInt(left.len)))); - const close_fraction = @as(f32, @floatFromInt(count_close)) / @as(f32, @floatFromInt(left.len)); - - const report: CompareReport = .{ - .nan_or_inf = nan_or_inf, - .max_absolute_error = max_absolute_error, - .mean_absolute_error = mean_absolute_error, - .rmse = rmse, - .close_fraction = close_fraction, - .p50_absolute_error = q(absolute_errors, 0.5), - .p90_absolute_error = q(absolute_errors, 0.9), - .p99_absolute_error = q(absolute_errors, 0.99), - .p999_absolute_error = q(absolute_errors, 0.999), + pub const exact_match: CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .epsilon_relative = 0, + .minimum_close_fraction = 1.00, }; - // TODO: Move - //const ok = !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; - return report; + pub const Report = struct { + nan_or_inf: bool, + max_absolute_error: f32, + mean_absolute_error: f32, + rmse: f32, + + close_fraction: f32, + + p50_absolute_error: f32, + p90_absolute_error: f32, + p99_absolute_error: f32, + p999_absolute_error: f32, + + pub fn ok(report: Report, opts: CompareOpts) bool { + return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; + } + + fn compareSlices(comptime float_cmp: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !CompareOpts.Report { + var left_iter = (&left).contiguousItemsIterator(L); + var right_iter = (&right).contiguousItemsIterator(R); + var left_chunk: []const L = &.{}; + var right_chunk: []const R = &.{}; + var acc: Accumulator = if (comptime float_cmp) try .init(allocator, left.shape.count()) else .initNoQuantiles(left.shape.count()); + defer acc.deinit(allocator); + + while (true) { + if (left_chunk.len == 0) { + left_chunk = left_iter.next() orelse &.{}; + } + if (right_chunk.len == 0) { + right_chunk = right_iter.next() orelse &.{}; + } + + if (left_chunk.len == 0 or right_chunk.len == 0) { + std.debug.assert(left_chunk.len == right_chunk.len); + return acc.report(); + } + + const n = @min(left_chunk.len, right_chunk.len); + if (comptime float_cmp) { + acc.cmpFloatSlices(L, R, left_chunk[0..n], right_chunk[0..n], opts); + } else { + acc.cmpIntSlices(L, left_chunk[0..n], right_chunk[0..n]); + } + + left_chunk = left_chunk[n..]; + right_chunk = right_chunk[n..]; + } + } + + pub fn format( + self: Report, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print( + \\ nan_or_inf: {} + \\ max_absolute_error: {d} + \\ mean_absolute_error: {d} + \\ rmse: {d} + \\ close_fraction: {d} + \\ p50_absolute_error: {d} + \\ p90_absolute_error: {d} + \\ p99_absolute_error: {d} + \\ p999_absolute_error: {d} + , .{ + self.nan_or_inf, + self.max_absolute_error, + self.mean_absolute_error, + self.rmse, + self.close_fraction, + self.p50_absolute_error, + self.p90_absolute_error, + self.p99_absolute_error, + self.p999_absolute_error, + }); + } + }; + + const Accumulator = struct { + absolute_errors: []f32, + relative_errors: []f32, + expected_total_count: usize, + processed_count: usize = 0, + nan_or_inf: bool = false, + max_absolute_error: f32 = 0, + sum_absolute_error: f64 = 0, + sum_squared_error: f64 = 0, + count_close: usize = 0, + + fn init(allocator: std.mem.Allocator, count: usize) !Accumulator { + const absolute_errors = try allocator.alloc(f32, count); + errdefer allocator.free(absolute_errors); + const relative_errors = try allocator.alloc(f32, count); + + @memset(absolute_errors, 0); + @memset(relative_errors, 0); + return .{ + .expected_total_count = count, + .absolute_errors = absolute_errors, + .relative_errors = relative_errors, + }; + } + + fn initNoQuantiles(count: usize) Accumulator { + return .{ + .expected_total_count = count, + .absolute_errors = &.{}, + .relative_errors = &.{}, + }; + } + + fn deinit(self: *Accumulator, allocator: std.mem.Allocator) void { + allocator.free(self.absolute_errors); + allocator.free(self.relative_errors); + } + + fn cmpFloatSlices(self: *Accumulator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) void { + std.debug.assert(left.len == right.len); + std.debug.assert(self.processed_count + left.len <= self.absolute_errors.len); + + const start = self.processed_count; + for (left, right, 0..) |l, r, i| { + const l_f32 = zml.floats.floatCast(f32, l); + const r_f32 = zml.floats.floatCast(f32, r); + if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { + self.nan_or_inf = true; + continue; + } + const absolute_error: f32 = @abs(l_f32 - r_f32); + self.max_absolute_error = @max(self.max_absolute_error, absolute_error); + self.sum_absolute_error += absolute_error; + self.sum_squared_error += absolute_error * absolute_error; + + const scale = @max(@abs(l_f32), @abs(r_f32)); + const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; + if (absolute_error <= tolerance) { + self.count_close += 1; + } + + self.absolute_errors[start + i] = absolute_error; + const denom = @max(opts.epsilon_relative, scale); + self.relative_errors[start + i] = absolute_error / denom; + } + self.processed_count += left.len; + } + + fn cmpIntSlices(self: *Accumulator, comptime T: type, left: []const T, right: []const T) void { + std.debug.assert(left.len == right.len); + + for (left, right) |l, r| { + switch (@typeInfo(T)) { + .int, .float => self.count_close += if (l == r) 1 else 0, + .vector => |vec| self.count_close += @popCount(@as(@Int(.unsigned, vec.len), @bitCast(l == r))), + else => @panic("cmpIntSlices expect int or vector of int"), + } + } + self.processed_count += left.len; + } + + fn report(self: *Accumulator) CompareOpts.Report { + std.debug.assert(self.processed_count == self.expected_total_count); + if (self.processed_count == 0) { + return .{ + .nan_or_inf = false, + .max_absolute_error = 0, + .mean_absolute_error = std.math.nan(f32), + .rmse = std.math.nan(f32), + .close_fraction = std.math.nan(f32), + .p50_absolute_error = 0, + .p90_absolute_error = 0, + .p99_absolute_error = 0, + .p999_absolute_error = 0, + }; + } + + std.sort.heap(f32, self.absolute_errors, {}, std.sort.asc(f32)); + std.sort.heap(f32, self.relative_errors, {}, std.sort.asc(f32)); + + const q = struct { + fn q(values: []const f32, frac: f32) f32 { + if (values.len == 0) return 0; + const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); + return values[idx]; + } + }.q; + + const mean_absolute_error = stdx.math.divFloat(f64, self.sum_absolute_error, self.processed_count); + const rmse = std.math.sqrt(stdx.math.divFloat(f64, self.sum_squared_error, self.processed_count)); + const close_fraction = stdx.math.divFloat(f64, self.count_close, self.processed_count); + + return .{ + .nan_or_inf = self.nan_or_inf, + .max_absolute_error = self.max_absolute_error, + .mean_absolute_error = @floatCast(mean_absolute_error), + .rmse = @floatCast(rmse), + .close_fraction = @floatCast(close_fraction), + .p50_absolute_error = q(self.absolute_errors, 0.5), + .p90_absolute_error = q(self.absolute_errors, 0.9), + .p99_absolute_error = q(self.absolute_errors, 0.99), + .p999_absolute_error = q(self.absolute_errors, 0.999), + }; + } + }; +}; + +pub fn compareFloats(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareOpts.Report { + var acc: CompareOpts.Accumulator = try .init(allocator, left.len); + defer acc.deinit(allocator); + acc.cmpFloatSlices(true, L, R, left, right, opts); + return acc.report(); } pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { @@ -238,25 +369,6 @@ fn BufferizedWithArgs(comptime T: type) type { return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } -/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. -/// This helper can be used in tests to make the code a bit less verbose. -/// It doesn't handle pointers in inputs/outputs structs. -pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - args.set(inputs); - exe.callOpts(io, args, &results, .{ .wait = true }); - - var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; - results.fill(.{&output}); - - return output; -} - fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); @@ -289,13 +401,14 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), + meshes: []const *const zml.Mesh, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const forward = @field(@TypeOf(layer), @tagName(func)); - const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); + const ArgsT = stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(forward))); var args: ArgsT = undefined; @@ -318,12 +431,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createTensor(subkey); + tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); ctx_.index += 1; } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args); + const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); @@ -340,7 +453,7 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ .dma_chunks = 1, .dma_chunk_size = 4096, .store = activation_store.store, .parallelism = 1 }); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); @@ -349,17 +462,17 @@ pub fn testLayer( exe_args.set(.{ layer_weights, args_buffers }); - exe.callOpts(io, exe_args, &exe_results, .{ .wait = true }); - var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); + exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); + defer for (results) |*result| result.deinit(); var failed: bool = false; var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { - const expected_slice = b: { + const expected_slice = s: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); var reader = try store_output.getReader(subkey, io, &reader_buffer); @@ -368,7 +481,7 @@ pub fn testLayer( const expected_slice: zml.Slice = try .alloc(allocator, shape); errdefer expected_slice.free(allocator); try reader.interface.readSliceAll(expected_slice.data()); - break :b expected_slice; + break :s expected_slice; }; defer expected_slice.free(allocator); diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index 691bd0ee..f2cbc132 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -3,14 +3,14 @@ load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", - srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:zig", "//stdx", - "//zml/tokenizer/hftokenizers", + "//zml/tokenizer/iree", "//zml/tokenizer/sentencepiece", ], ) @@ -27,15 +27,13 @@ zig_binary( zig_test( name = "test", - main = "homemade.zig", - visibility = ["//visibility:public"], deps = [ - "//ffi:zig", - "//stdx", + ":tokenizer", ], ) -zig_srcs( - name = "sources", - zig_lib = ":tokenizer", -) +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":tokenizer", +# ) diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel deleted file mode 100644 index a07d855b..00000000 --- a/zml/tokenizer/hftokenizers/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_rust//rust:defs.bzl", "rust_static_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -rust_static_library( - name = "hftokenizers_rs", - srcs = ["hftokenizers.rs"], - crate_name = "zml_tokenizer_hftokenizers", - edition = "2021", - deps = ["@crates//:tokenizers"], -) - -cc_library( - name = "hftokenizers_cc", - hdrs = ["hftokenizers.h"], - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_rs", - "//ffi:cc", - ], -) - -zig_library( - name = "hftokenizers", - main = "hftokenizers.zig", - visibility = ["//visibility:public"], - deps = [ - ":hftokenizers_cc", - "//ffi:zig", - "//stdx", - ], -) diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock deleted file mode 100644 index 7164e77b..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.lock +++ /dev/null @@ -1,713 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", - "zerocopy 0.8.26", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bitflags" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - -[[package]] -name = "cc" -version = "1.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "compact_str" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "darling" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.20.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" -dependencies = [ - "darling_core", - "quote", - "syn", -] - -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" -dependencies = [ - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "macro_rules_attribute" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" -dependencies = [ - "monostate-impl", - "serde", -] - -[[package]] -name = "monostate-impl" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "onig" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy 0.7.35", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" -dependencies = [ - "either", - "itertools", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.216" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.134" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" -dependencies = [ - "ahash", - "aho-corasick", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "getrandom", - "itertools", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand", - "rayon", - "rayon-cond", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive 0.8.26", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -dependencies = [ - "tokenizers", -] diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml deleted file mode 100644 index f905c004..00000000 --- a/zml/tokenizer/hftokenizers/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "zml_tokenizer_hftokenizers" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokenizers = { version = "0.22.2", default-features = false, features = [ - # We don't need/want the progressbar feature - "onig", - "esaxx_fast", -] } - -[lib] -name = "zml_tokenizer_hftokenizers" -path = "hftokenizers.rs" diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h deleted file mode 100644 index f779b575..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include -#include - -#include "ffi/zig_slice.h" - -typedef struct hftokenizers hftokenizers; - -hftokenizers *hftokenizers_new(zig_slice); -hftokenizers *hftokenizers_new_from_bytes(zig_slice); -void hftokenizers_drop(hftokenizers *tokenizer); -zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); -void hftokenizers_tokens_drop(zig_slice tokens); -zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); -void hftokenizers_str_drop(zig_slice text); -uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs deleted file mode 100644 index 048283cc..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[repr(C)] -struct ZigSlice { - ptr: *mut T, - len: usize, -} - -impl ZigSlice { - fn as_slice(&self) -> &[T] { - unsafe { std::slice::from_raw_parts(self.ptr, self.len) } - } - - fn as_slice_mut(&self) -> &mut [T] { - unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } - } -} - -#[no_mangle] -extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_file(std::path::Path::new( - std::str::from_utf8(path.as_slice()).unwrap(), - )) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { - return Box::into_raw(Box::new( - tokenizers::Tokenizer::from_bytes(bytes.as_slice()) - .unwrap(), - )); -} - -#[no_mangle] -extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { - drop(unsafe { Box::from_raw(t) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_encode( - t: *mut tokenizers::Tokenizer, - string: ZigSlice, -) -> ZigSlice { - let input_str = std::str::from_utf8(string.as_slice()).unwrap(); - let encoded = unsafe { t.as_ref() } - .unwrap() - .encode_fast(input_str, false) - .unwrap(); - - // Convert the result to a boxed slice - let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: ids.as_mut_ptr(), - len: ids.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(ids); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { - // Reconstruct the Box from the zig slice so that it's dropped. - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_decode( - t: *mut tokenizers::Tokenizer, - ids: ZigSlice, -) -> ZigSlice { - let decoded = unsafe { t.as_ref() } - .unwrap() - .decode(ids.as_slice(), false) - .unwrap(); - - // Convert the result to a boxed slice - let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); - - // Retrieve the zig slice associated to the boxed slice. - let slice = ZigSlice { - ptr: string.as_mut_ptr(), - len: string.len(), - }; - - // Leak the box so that it's not deallocated. - Box::leak(string); - - return slice; -} - -#[no_mangle] -extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { - drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); -} - -#[no_mangle] -extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { - let id = unsafe { t.as_ref() } - .unwrap() - .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) - .unwrap_or(u32::MAX); - return id; -} diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig deleted file mode 100644 index 0871d6f9..00000000 --- a/zml/tokenizer/hftokenizers/hftokenizers.zig +++ /dev/null @@ -1,120 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const ffi = @import("ffi"); -const stdx = @import("stdx"); - -pub const Encoder = struct { - inner: *HFTokenizer, - current_ids: ?[]const u32 = null, - - fn init(inner: *HFTokenizer) Encoder { - return .{ .inner = inner }; - } - - pub fn reset(self: *Encoder) void { - if (self.current_ids) |current_ids_| { - c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); - self.current_ids = null; - } - } - - pub fn deinit(self: *Encoder) void { - self.reset(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); - return self.ids(); - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids orelse &.{}; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *HFTokenizer, - current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, - - fn init(inner: *HFTokenizer) Decoder { - return .{ .inner = inner }; - } - - pub fn deinit(self: *Decoder) void { - self.reset(); - } - - pub fn reset(self: *Decoder) void { - if (self.current_string) |current_string_| { - c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); - self.current_string = null; - } - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - self.reset(); - self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); - return self.string(); - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse &.{}; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -pub const HFTokenizer = opaque { - pub fn fromFile(model: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); - } - - pub fn fromBytes(bytes: []const u8) !*HFTokenizer { - return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); - } - - pub fn deinit(self: *HFTokenizer) void { - return c.hftokenizers_drop(@ptrCast(self)); - } - - pub fn encoder(self: *HFTokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *HFTokenizer) !Decoder { - return Decoder.init(self); - } - - pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { - const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); - return if (id == std.math.maxInt(u32)) null else id; - } -}; diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig deleted file mode 100644 index bd16d183..00000000 --- a/zml/tokenizer/homemade.zig +++ /dev/null @@ -1,1242 +0,0 @@ -//! Text tokenizer implementations -//! Disclaimer this is not a very robust implementation: -//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. -//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. -const std = @import("std"); -const testing = std.testing; -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/tokenizer"); - -/// Byte Pair Encoding tokenizer generally used for LLM. -pub const Tokenizer = struct { - tokens: [][]const u8, - token_lookup: std.StringHashMapUnmanaged(u32), - special_tokens: SpecialTokens, - - scores: []f32, - max_token_len: u32, - normalizer: ?Normalizer, - // Allows to split unknown unicode characters into bytes. - byte_fallback: bool = false, - - arena_state: std.heap.ArenaAllocator, - vocab_size: u32, - next_token_id: u32 = 0, - - pub const SpecialTokens = struct { - eos: u32, - bos: u32, - unk: u32, - pad: u32 = std.math.maxInt(u32), - hard_space: u32 = std.math.maxInt(u32), - }; - - pub fn init( - allocator: std.mem.Allocator, - vocab_size: u32, - max_token_len: u32, - normalizer: ?Normalizer, - special_tokens: SpecialTokens, - alloc_tokens: bool, - ) !Tokenizer { - var arena_state = std.heap.ArenaAllocator.init(allocator); - errdefer arena_state.deinit(); - const arena = arena_state.allocator(); - - var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; - errdefer token_lookup.deinit(arena); - - try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); - - const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]const u8, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(tokens); - - const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(scores); - - return .{ - .tokens = tokens, - .scores = scores, - .max_token_len = max_token_len, - .token_lookup = token_lookup, - .arena_state = arena_state, - .normalizer = normalizer, - .vocab_size = vocab_size, - .special_tokens = special_tokens, - }; - } - - pub fn deinit(self: Tokenizer) void { - self.arena_state.deinit(); - } - - pub fn encoder(self: *Tokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *Tokenizer) !Decoder { - return Decoder.init(self); - } - - /// Reads a new word directly into the tokenizer arena. - pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { - const arena = self.arena_state.allocator(); - - const token = try arena.alloc(u8, len); - try tok_reader.readSliceAll(token); - return self.addOwnedToken(score, token); - } - - /// Adds a new token (and copy it) - pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { - const arena = self.arena_state.allocator(); - return self.addOwnedToken(score, try arena.dupe(u8, token)); - } - - /// Adds a new token (without copying it) - pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { - const i = self.next_token_id; - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = i; - } - } - - pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = @intCast(i); - } - } - - pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { - return self.token_lookup.get(str); - } - - pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { - return self.token_lookup.get(token); - } - - pub const EncodeOptions = struct { - /// Should the beginning of sentence '' token be added. - add_bos: bool = true, - add_eos: bool = false, - pad_to: u32 = 0, - // Print tokenization intermediary steps. - debug: bool = false, - }; - - pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { - if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); - const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; - defer if (self.normalizer) |_| allocator.free(input); - if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); - - // Allocate a buffer that can fit all indices as well as extra character if requested. - // We then slice it so that the token merging code doesn't see the bos token. - const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); - const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; - - const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; - const mergeable = try allocator.alloc(MergeState, tok_buff.len); - - var num_tokens: usize = 0; - var it: CharTokenIterator = .{ .input = input }; - while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { - tok_buff[num_tokens] = token; - mergeable[num_tokens] = if (token == self.special_tokens.hard_space) - .hard_space - else - .idk; - } - - var stable_prefix: usize = 0; - var stable_off: usize = 0; - while (true) { - // This code is a bit overcomplicated cause I'm abstracting over two algorithms: - // BPE and sentencepiece unigram model. - // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. - // With unigram model you work at sentence level and you handle spaces as you would any other bytes, - // hoping the final tokens mostly align with spaces. - // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE - // by detecting when the first "word" is treated and can be safely removed from sequence. - // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) - // and have custom splitting rules. - // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases - // and are only using this for tinyllama/gguf models. - // If we come back to use this in production, the implementation would gain in speed/clarity - // by splitting in two. - // The merging token logic isn't that complicated anyway. - - // Step by step visualization of the progress. - if (options.debug) { - var _debug_buf: [256]u8 = undefined; - var debug_progress: std.Io.Writer = .fixed(&_debug_buf); - self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; - log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); - } - var best_score: f32 = -1e10; - var best_token: u32 = 0; - var best_idx: ?usize = null; - var input_off: usize = stable_off; - - // Find best tokens to merge in all available tokens - for (stable_prefix..num_tokens - 1) |i| { - if (tok_buff[i] == self.special_tokens.unk) { - input_off += 1; - continue; - } - const cur_tok = self.tokens[tok_buff[i]]; - defer input_off += cur_tok.len; - - // Lookup merge for current token, if not already done. - switch (mergeable[i]) { - .nope => continue, - .ready => {}, - .hard_space => { - // Since tokens are not allowed to merge through hard sep, - // we don't need to merge the sentence-wide best token. - // We can just merge the best token since beginning. - if (best_idx != null) break; - // OTOH if there was no merge possible since beginning, - // we can skip the beginning in future iterations. - stable_prefix = i + 1; - stable_off = input_off + cur_tok.len; - continue; - }, - .idk => { - - // Special tokens can't be concatenated. - if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { - // Detects memory corruption of tokens. - if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); - - if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { - log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); - @panic("invalid tokenization"); - } - } - const next_tok = self.tokens[tok_buff[i + 1]]; - // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. - const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; - const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; - // Save the result - mergeable[i] = if (self.lookup(concat_tokens)) |tok| - .{ .ready = tok } - else - .nope; - }, - } - - switch (mergeable[i]) { - .idk, .hard_space => unreachable, - .nope => continue, - .ready => |tok| { - if (self.scores[tok] > best_score) { - best_score = self.scores[tok]; - best_token = tok; - best_idx = i; - } - }, - } - } - - if (best_idx) |bidx| { - // Apply the merge. - tok_buff[bidx] = best_token; - std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); - std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); - num_tokens -= 1; - // We got two new merge lookups to do. - mergeable[bidx] = .idk; - if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; - } else { - // No merge candidate => we are done ! - break; - } - } - - if (options.add_eos) { - tok_buff[num_tokens] = self.special_tokens.eos; - num_tokens += 1; - } - if (options.add_bos) { - tok_buff_alloc[0] = self.special_tokens.bos; - num_tokens += 1; - } - if (num_tokens < options.pad_to) { - for (num_tokens..options.pad_to) |i| { - tok_buff_alloc[i] = self.special_tokens.pad; - } - num_tokens = options.pad_to; - } - - // Release extra memory we don't need anymore. - allocator.free(mergeable); - _ = allocator.resize(tok_buff_alloc, num_tokens); - return tok_buff_alloc[0..num_tokens]; - } - - /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. - pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { - return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) - "" - else if (id == self.special_tokens.unk) - "" - else if (id > self.tokens.len) - "" // this means we received an invalid id, but we didn't want to panic. - else - self.tokens[id]; - } - - /// Converts the given slice of tokens back into bytes. - /// Note that if the tokenizer allows sub-unicode bytes, it's possible - /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try self.decodeWithOpts(input, writer, .{}); - } - - pub fn decodeWithOpts( - self: *const Tokenizer, - input: []const u32, - output: *std.Io.Writer, - opts: struct { sep: []const u8 = "" }, - ) std.Io.Writer.Error!void { - const escaped = if (self.normalizer) |n| n.escapedSpace() else null; - // Flag used to indicate if the first dummy whitespace has been consumed. - var first_token: bool = true; - for (input) |id| { - // Retrieve the slice corresponding to the id. - var piece = self.lookupPiece(id); - - // Convert `▁` to a regular space. - if (escaped) |escspc| { - // we modify piece inside the loop, so we can use it in the condition - while (std.mem.startsWith(u8, piece, escspc)) { - piece = piece[escspc.len..]; - // don't output a space at beginning of text. - if (!first_token) try output.writeByte(' '); - } - } - - try output.writeAll(piece); - first_token = false; - try output.writeAll(opts.sep); - } - } - - /// Some tokenizers have bytes encoded in hex like this: "<0x40>". - /// This break the tokenization algorithm because the input text - /// will contain "@" not "<0x40>", - /// and if the input contains "<0x40>" it needs to not be treated as a single byte. - /// So we replace byte fallbacks strings, by their corresponding character. - /// This enables the normal tokenization algorithm to work. - pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { - tokenizer.byte_fallback = true; - var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); - var byte_fallback_buf = "<0x00>".*; - - for (0..256) |i| { - const c: u8 = @truncate(i); - single_bytes[i] = c; - - // First lookup the byte fallback entry. - // Note: we assume upper case, but we could try both upper and lower case if needed. - var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); - try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); - const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { - log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); - return error.InvalidInput; - }; - - // Check if the character is already present in the vocab. - // In that case, nothing to do, - // but note that the fallback token will be "unreachable", - // ie there is no way the tokenizer can produce it. - if (tokenizer.token_lookup.get(&.{c})) |_| continue; - - const idx: u32 = entry.value_ptr.*; - tokenizer.token_lookup.removeByPtr(entry.key_ptr); - tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); - } - } -}; - -test Tokenizer { - const allocator = std.testing.allocator; - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - - var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); - defer tokenizer.deinit(); - - try tokenizer.addToken(10, "hello"); - try tokenizer.addToken(3.5, "world"); - - try testing.expect(tokenizer.lookup("hello") == 0); - try testing.expect(tokenizer.lookup("world") == 1); - - // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto -} - -pub const Encoder = struct { - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - current_ids: []const u32 = &.{}, - - fn init(inner: *Tokenizer) !Encoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn reset(self: *Encoder) void { - self.current_ids = &.{}; - std.debug.assert(self.arena.reset(.retain_capacity)); - } - - pub fn deinit(self: *Encoder) void { - self.arena.deinit(); - } - - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { - self.reset(); - const res = try self.inner.encode(self.arena.allocator(), input, .{ - .add_bos = true, - .add_eos = false, - .pad_to = 0, - // Print tokenization intermediary steps. - .debug = false, - }); - self.current_ids = res; - return res; - } - - pub fn ids(self: *const Encoder) []const u32 { - return self.current_ids; - } -}; - -pub const Decoder = struct { - const StringBuffer = stdx.BoundedArray(u8, 128); - const TokensIdsBuffer = stdx.BoundedArray(u32, 4); - - inner: *Tokenizer, - arena: std.heap.ArenaAllocator, - - current_string: ?[]const u8 = null, - last_string: StringBuffer = .{ .len = 0 }, - last_token_ids: TokensIdsBuffer = .{ .len = 0 }, - - fn init(inner: *Tokenizer) !Decoder { - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. - _ = try arena.allocator().alloc(u32, 4096); - std.debug.assert(arena.reset(.retain_capacity)); - return .{ .inner = inner, .arena = arena }; - } - - pub fn deinit(self: *Decoder) void { - self.arena.deinit(); - } - - pub fn reset(self: *Decoder) void { - std.debug.assert(self.arena.reset(.retain_capacity)); - self.current_string = null; - } - - pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { - // TODO: revisite tokenizer api to use std.Io.Writer. - self.reset(); - // Reuse the same warmup than in init, to maximize contiguous writes. - var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; - try self.inner.decode(ids, &arena_writer.writer); - self.current_string = arena_writer.written(); - return self.current_string.?; - } - - pub fn string(self: *const Decoder) []const u8 { - return self.current_string orelse ""; - } - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { - if (self.last_token_ids.len >= self.last_token_ids.capacity()) { - _ = self.last_token_ids.orderedRemove(0); - } - self.last_token_ids.appendAssumeCapacity(token_id); - const new_string = try self.decode(self.last_token_ids.constSlice()); - if (self.last_string.len == 0) { - self.last_string = try StringBuffer.fromSlice(new_string); - return new_string; - } - var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); - var it = view.iterator(); - while (it.nextCodepointSlice()) |cp| { - const start = it.i - cp.len; - if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { - const chunk = new_string[self.last_string.len - start ..]; - self.last_string = try StringBuffer.fromSlice(new_string); - return chunk; - } - } - return null; - } -}; - -/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. -/// The output of this can be used to initialize the tokenization algorithm. -/// Normally we split the input text into utf8 codepoint, -/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, -/// depending on the tokenizer configuration. -const CharTokenIterator = struct { - state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, - input: []const u8, - - fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { - if (self.input.len == 0) return null; - return switch (self.state) { - .by_byte => |*byte_left| { - const idx = tokenizer.lookup(self.input[0..1]) orelse { - // Normally this has been caught when calling `rewriteByteFallbackTokens`. - std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); - }; - - self.input = self.input[1..]; - byte_left.* -|= 1; - if (byte_left.* == 0) self.state = .by_codepoint; - return idx; - }, - .by_codepoint => { - // Try to lookup valid utf8 codepoint first. - const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); - if (self.input.len < utf8_len) return error.TruncatedInput; - if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { - self.input = self.input[utf8_len..]; - return idx; - } - - // Otherwise split in bytes if it's allowed. - if (tokenizer.byte_fallback) { - // TODO: replace this by a continue statement next time we bump Zig. - self.state = .{ .by_byte = utf8_len }; - return self.nextCodepointToken(tokenizer); - } - - // Or mark the full utf8 codepoint as unknown. - log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); - self.input = self.input[utf8_len..]; - return tokenizer.special_tokens.unk; - }, - }; - } -}; - -test CharTokenIterator { - const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; - var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); - defer tokenizer.deinit(); - - tokenizer.addOwnedToken(1.0, ""); // 0 - tokenizer.addOwnedToken(1.0, ""); // 1 - tokenizer.addOwnedToken(1.0, ""); // 2 - tokenizer.addOwnedToken(1.0, "ζ"); // 3 - tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte - tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte - tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte - tokenizer.addOwnedToken(1.0, "L"); // 7 - - // No byte fallback - { - tokenizer.byte_fallback = false; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); - } - - // with byte fallback - { - tokenizer.byte_fallback = true; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: stdx.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); - } -} - -/// Text normalizer. -/// Most tokenizer assumes the input text have been prepocessed with on of those. -pub const Normalizer = struct { - /// Space token used by sentencepiece derived tokenizer. - pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - - _whitespace: stdx.BoundedArray(u8, 8) = .{}, - - flags: packed struct { - remove_extra_whitespaces: bool, - add_dummy_prefix: bool, - add_dummy_suffix: bool, - /// Cheap lower casing. - /// TODO: try to match Python "lower" - lower_case_ascii: bool, - /// cheap ascii punct splitting. - // doing this processing ahead of time simplifies the logic - split_on_punct_ascii: bool, - }, - - pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { - var res: Normalizer = .{ .flags = flags }; - if (escaped_whitespace) |escaped| { - res._whitespace.appendSliceAssumeCapacity(escaped); - } - return res; - } - - pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { - return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; - } - - fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { - try normalized.appendSlice(allocator, data); - for (data) |_| try normalized_to_origin.append(allocator, consumed); - } - - pub const Result = struct { - /// Normalized string - normalized: []const u8, - /// Mapping between chars in the original string and chars in the new string - normalized_to_origin: []const usize, - - pub fn deinit(self: Result, allocator: std.mem.Allocator) void { - allocator.free(self.normalized); - allocator.free(self.normalized_to_origin); - } - }; - - /// Simplifed version of Sentencepiece normalizer. - /// - /// Llama2 uses a normalizer called "identity" so this basically only handles trailing - /// whitespaces and replaces whitespace with the "▁" (U+2581) character. - pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { - const res = try self.normalizeWithMapping(allocator, input); - allocator.free(res.normalized_to_origin); - return res.normalized; - } - - /// Returns both the normalized string and a mapping between the normalized string and the original. - pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { - // Number of bytes consumed from the input. - var consumed: usize = 0; - var trimmed_input = input; - - // Skip leading whitespaces. - if (self.flags.remove_extra_whitespaces) { - while (trimmed_input.len != 0) { - if (trimmed_input[0] != ' ') break; - trimmed_input = trimmed_input[1..]; - consumed += 1; - } - } - - // If the trimmed input is empty, we are done. - if (trimmed_input.len == 0) { - return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; - } - - // Pre-allocate outputs - const space = self.escapedSpace() orelse " "; - const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); - errdefer normalized.deinit(allocator); - var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); - errdefer normalized_to_origin.deinit(allocator); - - // If the spec asks for it, add a whitespace at the beginning. - if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - var is_prev_space: bool = true; - var is_prev_word: bool = false; - - while (trimmed_input.len != 0) { - // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. - // Since we have an "identity" normalizer, each slice is actually a unicode character. - const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); - var slice = trimmed_input[0..multibyte_length]; - const origin = consumed; - consumed += multibyte_length; - trimmed_input = trimmed_input[multibyte_length..]; - - if (self.flags.remove_extra_whitespaces and is_prev_space) { - while (slice.len > 0 and slice[0] == ' ') { - slice = slice[1..]; - } - if (slice.len == 0) continue; - } - is_prev_space = slice[slice.len - 1] == ' '; - - if (slice.len == 1) ascii: { - // The more advanced logic only works with ascii atm - var byte = slice[0]; - if (self.escapedSpace() != null and byte == ' ') { - // replace the space token by the special token - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - is_prev_word = false; - break :ascii; - } else if (self.flags.split_on_punct_ascii) { - if (is_prev_word and isPunct(slice)) { - // Insert a space, but continue handling the rest - try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); - } - } - if (self.flags.lower_case_ascii) { - byte = std.ascii.toLower(byte); - } - try normalized.append(allocator, byte); - try normalized_to_origin.append(allocator, origin); - } else { - // we can safely copy to the output. - try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); - } - is_prev_word = !is_prev_space and !isPunct(slice); - } - - // Skip trailing whitespaces - if (self.flags.remove_extra_whitespaces) { - while (std.mem.endsWith(u8, normalized.items, space)) { - const length = normalized.items.len - space.len; - consumed = normalized_to_origin.items[length]; - try normalized.resize(allocator, length); - try normalized_to_origin.resize(allocator, length); - } - } - - try normalized_to_origin.append(allocator, consumed); - - std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); - - if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); - - return .{ - .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), - .normalized = try normalized.toOwnedSlice(allocator), - }; - } - - pub fn wellKnown(impl: KnownImplementation) Normalizer { - return switch (impl) { - .sentencepiece => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, sentencepiece_space), - .llama3 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - .gpt2 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - }; - } - - pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { - var normalizer: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_suffix = false, - .add_dummy_prefix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - - // Normalizer config can be a single normalizer, or a sequence of normalizers. - const maybe_steps = objectGet(config, .array, "normalizers"); - const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; - - for (steps) |step_val| { - if (step_val != .object) { - return error.InvalidNormalizerJson; - } - const step = step_val.object; - - const step_type = objectGet(step, .string, "type") orelse { - return error.InvalidNormalizerJson; - }; - if (std.mem.eql(u8, "Prepend", step_type)) { - normalizer.flags.add_dummy_prefix = true; - } else if (std.mem.eql(u8, "Append", step_type)) { - normalizer.flags.add_dummy_suffix = true; - } else if (std.mem.eql(u8, "Replace", step_type)) { - const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; - const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; - - if (std.mem.eql(u8, str_pattern, " ")) { - normalizer._whitespace.appendSliceAssumeCapacity( - objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, - ); - } else { - log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); - } - } else { - log.warn("Unknown normalizer type: {s}", .{step_type}); - } - } - - return normalizer; - } - - test "Normalizer.fromHfJson" { - const config_json = - \\{ - \\ "type": "Sequence", - \\ "normalizers": [ - \\ { - \\ "type": "Prepend", - \\ "prepend": "▁" - \\ }, - \\ { - \\ "type": "Replace", - \\ "pattern": { - \\ "String": " " - \\ }, - \\ "content": "▁" - \\ } - \\ ] - \\} - ; - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); - const normalizer = try Normalizer.fromHfJson(config.object); - - const expected = Normalizer{ - ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, - .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - }; - try std.testing.expectEqual(expected.flags, normalizer.flags); - try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); - } -}; -pub const KnownImplementation = enum(u8) { - sentencepiece, - gpt2, - llama3, -}; - -fn isPunct(unicode_char: []const u8) bool { - // TODO use unicode categories - if (unicode_char.len > 1) return false; - - return switch (unicode_char[0]) { - ' ', '\t' => false, - 0...8 => true, - 10...31 => true, - '!'...'/' => true, - ':'...'@' => true, - '['...'`' => true, - '{'...'~' => true, - else => false, - }; -} - -test Normalizer { - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); - defer res.deinit(testing.allocator); - - try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); - try testing.expectEqualSlices( - usize, - // H e l l ŏ ␣ w o r l d ! - &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, - res.normalized_to_origin, - ); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, " Hello world! ", res); - } - - { - const n = Normalizer.init( - .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - Normalizer.sentencepiece_space, - ); - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = true, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "hello world! ", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = true, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "Hello world ! ", res); - } -} - -/// gpt2 had their own way of storing text. -/// Unfortunately this has contaminated other models. -/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, -/// into utf8 bytes, and do lookups at runtime. -pub const Gpt2TextDecoder = struct { - const Code = stdx.BoundedArray(u8, 2); - - // TODO: benchmark this is more efficient than doing the conversion at runtime. - code_to_byte: std.AutoArrayHashMap(Code, u8), - - pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { - var self = Gpt2TextDecoder{ - .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), - }; - try self.code_to_byte.ensureTotalCapacity(256); - errdefer unreachable; - - var n: usize = 0; - for (0..256) |index| { - var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init - const i: u8 = @intCast(index); - if (isPrintableByte(i)) { - if (std.ascii.isAscii(i)) { - code.appendAssumeCapacity(i); - } else { - const codepoint: u21 = @as(u21, @intCast(i)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - } - } else { - const codepoint: u21 = 256 + @as(u21, @intCast(n)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - n += 1; - } - - self.code_to_byte.putAssumeCapacityNoClobber(code, i); - } - return self; - } - - pub fn deinit(self: *Gpt2TextDecoder) void { - self.code_to_byte.deinit(); - } - - /// Transform bytes representing text under the gpt2 encoding, - /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { - var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; - while (it.nextCodepointSlice()) |codepoint| { - const code: Code = switch (codepoint.len) { - 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init - 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, - else => return error.InvalidInput, - }; - const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; - try writer.writeByte(byte); - } - } - - inline fn isPrintableByte(c: u8) bool { - return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); - } -}; - -test Gpt2TextDecoder { - var decoder = try Gpt2TextDecoder.init(testing.allocator); - defer decoder.deinit(); - - var buf: [128]u8 = undefined; - - // Ascii is not changed. - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("getTitle", &out); - try testing.expectEqualStrings("getTitle", out.buffered()); - } - - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("Ċ", &out); - try testing.expectEqualStrings("\n", out.buffered()); - } - - // Leading space are represented with 'Ġ' - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠUINavigationController", &out); - try testing.expectEqualStrings(" UINavigationController", out.buffered()); - } - // Russian is wild - { - var out: std.Io.Writer = .fixed(&buf); - try decoder.decode("ĠÑĢабоÑĤ", &out); - try testing.expectEqualStrings(" работ", out.buffered()); - } -} - -/// Open a json file in HF format and load the vocab from it. -pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { - const file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer file.close(); - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const file_content = b: { - var reader = file.reader(io, &.{}); - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - _ = try reader.interface.streamRemaining(&writer.writer); - break :b try writer.toOwnedSlice(); - }; - defer allocator.free(file_content); - - const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ - .duplicate_field_behavior = .use_last, - }); - const main_object = switch (info) { - .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { - return error.InvalidFormat; - } else obj, - else => return error.InvalidFormat, - }; - - const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; - const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; - const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; - const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); - - const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| - try Normalizer.fromHfJson(normalizer_config) - else - Normalizer.wellKnown(.llama3); - - // delay init of special tokens. - var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); - errdefer tokenizer.deinit(); - - // Buffer containing all concatenated tokens. - // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); - const original_alloc = all_tokens.writer.buffer.ptr; - // A re-alloc event here means we have invalidated all slices inside the tokenizer. - // If this is too annoying we could switch to a custom type instead of slices. - defer { - std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); - } - - // gpt2 based tokenizer got a special way of encoding unicode. - // we don't know in advance if this will be used by this tokenizer or not. - // so we assume it is the case, but if we find some unicode character, - // outside of the range used by gpt2 we know it was wrong, and start over. - var is_gpt2_vocab: bool = true; - var gpt2_decoder = try Gpt2TextDecoder.init(allocator); - defer gpt2_decoder.deinit(); - var it = vocab.iterator(); - while (it.next()) |kv| { - const n = all_tokens.writer.buffered().len; - gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { - switch (err) { - error.InvalidInput => { - is_gpt2_vocab = false; - break; - }, - error.WriteFailed => return error.OutOfMemory, - } - }; - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - - if (!is_gpt2_vocab) { - // We where wrong, this is not a gpt2 vocab, start over, - // and reset the tokenizer state. - tokenizer.next_token_id = 0; - tokenizer.token_lookup.clearRetainingCapacity(); - all_tokens.clearRetainingCapacity(); - it = vocab.iterator(); - while (it.next()) |kv| { - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = try dup(&all_tokens, kv.key_ptr.*); - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - } - - // More tokens, typically added during fine tuning of the model. - for (added_tokens) |token_obj| { - if (token_obj != .object) return error.InvalidFormat; - const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; - const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); - const n = all_tokens.written().len; - try if (is_gpt2_vocab) - gpt2_decoder.decode(v, &all_tokens.writer) - else - all_tokens.writer.writeAll(v); - const token = all_tokens.written()[n..]; - tokenizer.addOwnedTokenByIndex(id, 0, token); - } - // We won't add more tokens here, let release. - _ = try all_tokens.toOwnedSlice(); - - var unk = tokenizer.lookup(""); - if (objectGet(model, .integer, "unk_token")) |unk_tok| { - unk = @intCast(unk_tok); - } - - tokenizer.special_tokens = .{ - // TODO allow users to specify special tokens or read them from a tokenizer_config.json file - .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), - .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), - .unk = unk orelse std.math.maxInt(u32), - }; - - const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; - if (!byte_fallback and unk == null) { - // GPT2 tokenizer have byte fallback already encoded in the model, - // but the json generally don't have the field set. - // We can detect it though because they don't specify an unknown token. - if (is_gpt2_vocab) { - tokenizer.byte_fallback = true; - } else { - log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); - } - } else if (byte_fallback) { - try tokenizer.rewriteByteFallbackTokens(); - } - return tokenizer; -} - -/// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { - const n = allocating.written().len; - try allocating.writer.writeAll(str); - return allocating.written()[n..]; -} - -/// Returns the given entry in a json object only if it has the right type. -fn objectGet( - object: std.json.ObjectMap, - comptime kind: std.meta.FieldEnum(std.json.Value), - key: []const u8, -) ?@FieldType(std.json.Value, @tagName(kind)) { - const val = object.get(key) orelse return null; - if (val != kind) return null; - return @field(val, @tagName(kind)); -} - -pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { - const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer tokenizer_file.close(); - var read_buff: [4096]u8 = undefined; - var tok_reader = tokenizer_file.reader(io, &read_buff); - const r: *std.Io.Reader = &tok_reader.interface; - - const max_token_len = try readValueLE(u32, r); - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); - var i: u32 = 0; - while (readToken(&tokenizer, r)) : (i += 1) { - // Pass - } else |_| { - if (i < vocab_size) { - log.info("Read {d} words out of {d}", .{ i, vocab_size }); - } - tokenizer.vocab_size = i; - } - try tokenizer.rewriteByteFallbackTokens(); - return tokenizer; -} - -fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { - const score: f32 = try readValueLE(f32, tok_reader); - const len: usize = try readValueLE(u32, tok_reader); - try tokenizer.readTokenInto(score, len, tok_reader); -} - -fn readValueLE(T: type, reader: *std.Io.Reader) !T { - var res: [1]T = undefined; - try reader.readSliceEndian(T, &res, .little); - return res[0]; -} diff --git a/zml/tokenizer/iree/BUILD.bazel b/zml/tokenizer/iree/BUILD.bazel new file mode 100644 index 00000000..a49ccd61 --- /dev/null +++ b/zml/tokenizer/iree/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "iree", + srcs = ["tokenizer.zig"], + main = "tokenizer.zig", + visibility = ["//visibility:public"], + deps = [ + "@iree//runtime/src/iree/tokenizer", + "@iree//runtime/src/iree/tokenizer:normalizer", + "@iree//runtime/src/iree/tokenizer/format/huggingface:normalizer_json", + "@iree//runtime/src/iree/tokenizer/format/huggingface:tokenizer_json", + "@iree//runtime/src/iree/tokenizer/normalizer:lowercase", + "@iree//runtime/src/iree/tokenizer/normalizer:nfc", + "@iree//runtime/src/iree/tokenizer/normalizer:nfd", + "@iree//runtime/src/iree/tokenizer/normalizer:nfkd", + "@iree//runtime/src/iree/tokenizer/normalizer:regex_replace", + "@iree//runtime/src/iree/tokenizer/normalizer:replace", + "@iree//runtime/src/iree/tokenizer/normalizer:sequence", + "@iree//runtime/src/iree/tokenizer/normalizer:strip", + "@iree//runtime/src/iree/tokenizer/normalizer:strip_accents", + "@iree//runtime/src/iree/tokenizer/vocab", + ], +) + +zig_test( + name = "test", + deps = [ + ":iree", + ], +) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig new file mode 100644 index 00000000..b1586afb --- /dev/null +++ b/zml/tokenizer/iree/tokenizer.zig @@ -0,0 +1,1398 @@ +const std = @import("std"); + +const c = @import("c"); + +const log = std.log.scoped(.@"zml/tokenizer"); + +pub const Error = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +const StatusCodeMask = 0x1F; + +inline fn stringView(s: []const u8) c.iree_string_view_t { + return .{ .data = s.ptr, .size = s.len }; +} + +inline fn mutableStringView(s: []u8) c.iree_mutable_string_view_t { + return .{ .data = s.ptr, .size = s.len }; +} + +inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { + return .{ .data = bytes.ptr, .data_length = bytes.len }; +} + +inline fn tokenOutput(ids: []u32) c.iree_tokenizer_token_output_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } + return .{ + .capacity = ids.len, + .token_ids = @ptrCast(ids.ptr), + .token_offsets = null, + .type_ids = null, + }; +} + +inline fn tokenIdList(ids: []const u32) c.iree_tokenizer_token_id_list_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } + return .{ .count = ids.len, .values = @ptrCast(ids.ptr) }; +} + +inline fn statusCode(status: c.iree_status_t) u32 { + return @intCast(@intFromPtr(status.?) & StatusCodeMask); +} + +fn checkOk(status: c.iree_status_t) !void { + if (status == null) return; + const code = statusCode(status); + var buffer: [1024]u8 = undefined; + var buffer_length: usize = 0; + if (c.iree_status_format(status, buffer.len, &buffer, &buffer_length)) { + log.err("IREE tokenizer error: {s}", .{buffer[0..@min(buffer_length, buffer.len - 1)]}); + } + _ = c.iree_status_ignore(status); + return switch (code) { + c.IREE_STATUS_CANCELLED => Error.Cancelled, + c.IREE_STATUS_UNKNOWN => Error.Unknown, + c.IREE_STATUS_INVALID_ARGUMENT => Error.InvalidArgument, + c.IREE_STATUS_DEADLINE_EXCEEDED => Error.DeadlineExceeded, + c.IREE_STATUS_NOT_FOUND => Error.NotFound, + c.IREE_STATUS_ALREADY_EXISTS => Error.AlreadyExists, + c.IREE_STATUS_PERMISSION_DENIED => Error.PermissionDenied, + c.IREE_STATUS_RESOURCE_EXHAUSTED => Error.ResourceExhausted, + c.IREE_STATUS_FAILED_PRECONDITION => Error.FailedPrecondition, + c.IREE_STATUS_ABORTED => Error.Aborted, + c.IREE_STATUS_OUT_OF_RANGE => Error.OutOfRange, + c.IREE_STATUS_UNIMPLEMENTED => Error.Unimplemented, + c.IREE_STATUS_INTERNAL => Error.Internal, + c.IREE_STATUS_UNAVAILABLE => Error.Unavailable, + c.IREE_STATUS_DATA_LOSS => Error.DataLoss, + c.IREE_STATUS_UNAUTHENTICATED => Error.Unauthenticated, + else => Error.Internal, + }; +} + +pub const Tokenizer = struct { + // TODO: use the ArrayList pattern. Stop copying allocator everywhere + allocator: std.mem.Allocator, + inner: *c.iree_tokenizer_t, + + pub fn fromHuggingFaceJson(allocator: std.mem.Allocator, json: []const u8) !Tokenizer { + // TODO: wrap allocator into an iree allocator + var raw: ?*c.iree_tokenizer_t = null; + try checkOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); + return .{ + .allocator = allocator, + .inner = raw.?, + }; + } + + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + return fromHuggingFaceJson(allocator, bytes); + } + + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + const json = try std.Io.Dir.cwd().readFileAlloc(io, model, allocator, .unlimited); + defer allocator.free(json); + return try fromHuggingFaceJson(allocator, json); + } + + pub fn deinit(self: *Tokenizer) void { + c.iree_tokenizer_free(self.inner); + } + + pub fn encoder(self: *const Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *const Tokenizer) !Decoder { + return Decoder.init(self, self.allocator); + } + + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return Normalizer.init(self.allocator, sequence); + } + + pub fn normalizerFromHuggingFaceJson(_: *const Tokenizer, json: []const u8) !Normalizer { + return Normalizer.fromHuggingFaceJson(json); + } + + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + // Looks up a string in the vocabulary. + // Returns the token ID if found, or -1 if not found. + const id = c.iree_tokenizer_vocab_lookup(vocab, stringView(token)); + return if (id < 0) null else @intCast(id); + } + + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + if (token_id > std.math.maxInt(i32)) return null; + + const text = c.iree_tokenizer_vocab_token_text(vocab, @intCast(token_id)); + if (text.data == null) return null; + + return text.data[0..text.size]; + } + + pub const Encoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + transform_buffer: []u8, + state: *c.iree_tokenizer_encode_state_t, + + fn checkEncodeStatus(status: c.iree_status_t) !void { + // Output exhaustion is normal streaming backpressure. The caller + // commits any produced tokens and retries with more output space. + if (status != null and statusCode(status) == c.IREE_STATUS_RESOURCE_EXHAUSTED) { + _ = c.iree_status_ignore(status); + return Error.ResourceExhausted; + } + try checkOk(status); + } + + fn init(tokenizer: *const Tokenizer) !Encoder { + var state_size: usize = undefined; + try checkOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try tokenizer.allocator.alloc(u8, state_size); + errdefer tokenizer.allocator.free(state_storage); + + const transform_size: usize = c.IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE; + const transform_buffer = try tokenizer.allocator.alloc(u8, transform_size); + errdefer tokenizer.allocator.free(transform_buffer); + + var state: ?*c.iree_tokenizer_encode_state_t = null; + try checkOk(c.iree_tokenizer_encode_state_initialize( + tokenizer.inner, + byteSpan(state_storage), + byteSpan(transform_buffer), + c.iree_tokenizer_offset_run_list_empty(), + c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START, + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .transform_buffer = transform_buffer, + .state = state.?, + }; + } + + pub fn deinit(self: *Encoder) void { + c.iree_tokenizer_encode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + self.tokenizer.allocator.free(self.transform_buffer); + } + + pub fn reset(self: *Encoder) void { + c.iree_tokenizer_encode_state_reset(self.state, c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START); + } + + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { + var token_ids = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + errdefer token_ids.deinit(); + + try self.feed(text, &token_ids.writer); + try self.finalize(&token_ids.writer); + + const bytes = try token_ids.toOwnedSlice(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); + } + + /// /!\ Assumes the writer provides buffers with proper alignment. + /// /!\ Remember to call the finalize() method + pub fn writer(self: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return Writer.init(self, buffer, out); + } + + pub const Writer = struct { + encoder: *Encoder, + out: *std.Io.Writer, + interface: std.Io.Writer, + + fn init(encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return .{ + .encoder = encoder_, + .out = out, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn finalize(self: *Writer) std.Io.Writer.Error!void { + try self.interface.flush(); + self.encoder.finalize(self.out) catch |e| { + log.err("Finalize failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + if (w.end == 0) return; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + + if (w.end != 0) { + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + var total: usize = 0; + // Prefix chunks: written once each. + for (data[0 .. data.len - 1]) |chunk| { + try self.feed(chunk); + total += chunk.len; + } + // Last chunk is the splat pattern. + const pattern = data[data.len - 1]; + if (pattern.len == 0) return total; + for (0..splat) |_| { + try self.feed(pattern); + total += pattern.len; + } + return total; + } + + fn feed(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { + self.encoder.feed(chunk, self.out) catch |e| { + log.err("Feed failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + }; + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn feed(self: *Encoder, text: []const u8, out: *std.Io.Writer) !void { + var min_tokens: usize = @max(text.len / 4, 32); + var remaining = text; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + checkEncodeStatus(c.iree_tokenizer_encode_state_feed( + self.state, + stringView(remaining), + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens = @max(32, min_tokens -| produced); + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + continue; + } + + out.advance(produced * @sizeOf(u32)); + min_tokens = @max(32, min_tokens -| produced); + remaining = remaining[consumed..]; + } + } + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn finalize(self: *Encoder, out: *std.Io.Writer) !void { + var min_tokens: usize = 4; + while (true) { + var produced: usize = 0; + + checkEncodeStatus(c.iree_tokenizer_encode_state_finalize( + self.state, + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced * @sizeOf(u32)); + if (!c.iree_tokenizer_encode_state_has_pending(self.state)) break; + if (produced == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + } + } + } + + fn writableTokenSliceGreedy(out: *std.Io.Writer, min_tokens: usize) std.Io.Writer.Error![]u32 { + std.debug.assert(@mod(out.end, @sizeOf(u32)) == 0); + const bytes = try out.writableSliceGreedy(min_tokens * @sizeOf(u32)); + std.debug.assert(@mod(@intFromPtr(bytes.ptr), 4) == 0); + const aligned_len = bytes.len - @mod(bytes.len, @sizeOf(u32)); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..aligned_len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); + } + }; + + pub const Decoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + state: *c.iree_tokenizer_decode_state_t, + + fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { + var state_size: usize = undefined; + try checkOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try allocator.alloc(u8, state_size); + errdefer allocator.free(state_storage); + + var state: ?*c.iree_tokenizer_decode_state_t = null; + try checkOk(c.iree_tokenizer_decode_state_initialize( + tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(state_storage), + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .state = state.?, + }; + } + + pub fn deinit(self: *Decoder) void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + } + + pub fn reset(self: *Decoder) !void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + var state: ?*c.iree_tokenizer_decode_state_t = null; + try checkOk(c.iree_tokenizer_decode_state_initialize( + self.tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(self.state_storage), + &state, + )); + self.state = state.?; + } + + pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); + var consumed: usize = 0; + var produced: usize = 0; + try checkOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(&.{token_id}), + mutableStringView(out), + &consumed, + &produced, + )); + std.debug.assert(consumed == 1); + return out[0..produced]; + } + + pub fn finalize(self: *Decoder, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); + var produced: usize = 0; + try checkOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(out), + &produced, + )); + return out[0..produced]; + } + + pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { + var min_output: usize = @max(token_ids.len * 4, 128); + var remaining = token_ids; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + checkOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(remaining), + mutableStringView(try out.writableSliceGreedy(min_output)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output = @max(128, min_output -| produced); + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + @branchHint(.unlikely); + min_output *= 2; + std.debug.assert(min_output < 1 << 20); + continue; + } + + out.advance(produced); + min_output = @max(128, min_output -| produced); + remaining = remaining[consumed..]; + } + + min_output = 32; + while (true) { + var produced: usize = 0; + + checkOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(try out.writableSliceGreedy(min_output)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output *= 2; + std.debug.assert(min_output < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced); + break; + } + } + }; +}; + +pub const Normalizer = struct { + pub const Kind = union(enum) { + /// Unicode Normalization Form C. + NFC, + /// Unicode Normalization Form D. + NFD, + /// Unicode Normalization Form KD. + NFKD, + /// Removes Unicode combining marks. + /// filters out characters in the Unicode Mark category (Mn, Mc, Me). + StripAccents, + /// Converts text to lowercase. + Lowercase, + /// Substitutes occurrences of a literal or regular-expression pattern with the specified content. + Replace: struct { + kind: enum { + regex, + exact, + }, + pattern: []const u8, + content: []const u8, + }, + /// Removes leading and/or trailing whitespace. + Strip: enum(u8) { + left = 0b10, + right = 0b01, + both = 0b11, + }, + }; + + sequence: *c.iree_tokenizer_normalizer_t, + sequence_size: usize, + + // Supported normalizer types as of `4d4e97d00f099a21f38eeff26f82a6d9e3643a11`: + // - Sequence: Chains multiple normalizers in order + // - Lowercase: Unicode case folding + // - Strip: Leading/trailing whitespace removal + // - Prepend: Prefix string insertion + // - StripAccents: Removes combining marks (without NFD) + // - BertNormalizer: Combined BERT normalization pipeline + pub fn fromHuggingFaceJson(json: []const u8) !Normalizer { + var normalizer: ?*c.iree_tokenizer_normalizer_t = null; + errdefer if (normalizer) |ptr| c.iree_tokenizer_normalizer_free(ptr); + + try checkOk(c.iree_tokenizer_huggingface_parse_normalizer(.{ .data = json.ptr, .size = json.len }, c.iree_allocator_system(), &normalizer)); + if (normalizer == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer.?)); + + return .{ + .sequence = normalizer.?, + .sequence_size = sequence_size, + }; + } + + fn createNormalizer(normalizer_kind: Kind) !?*c.iree_tokenizer_normalizer_t { + var norm: ?*c.iree_tokenizer_normalizer_t = null; + + const iree_allocator = c.iree_allocator_system(); + + const status = switch (normalizer_kind) { + .NFC => c.iree_tokenizer_normalizer_nfc_allocate(iree_allocator, &norm), + .NFD => c.iree_tokenizer_normalizer_nfd_allocate(iree_allocator, &norm), + .NFKD => c.iree_tokenizer_normalizer_nfkd_allocate(iree_allocator, &norm), + .StripAccents => c.iree_tokenizer_normalizer_strip_accents_allocate(iree_allocator, &norm), + .Lowercase => c.iree_tokenizer_normalizer_lowercase_allocate(iree_allocator, &norm), + .Replace => |rule| switch (rule.kind) { + .regex => c.iree_tokenizer_normalizer_regex_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + .exact => c.iree_tokenizer_normalizer_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + }, + .Strip => |direction| b: { + const strip_left = @backingInt(direction) & 0b10 != 0; + const strip_right = @backingInt(direction) & 0b01 != 0; + break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); + }, + }; + + try checkOk(status); + if (norm == null) return error.NormalizerAllocationFailed; + return norm; + } + + pub fn init(allocator: std.mem.Allocator, sequence: []const Kind) !Normalizer { + if (sequence.len == 0) return error.EmptySequence; + + if (sequence.len == 1) { + // IREE requires a sequence of at least two normalizer. + const norm = try createNormalizer(sequence[0]) orelse unreachable; + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(norm)); + + return .{ + .sequence = norm, + .sequence_size = sequence_size, + }; + } + + var normalizers: std.ArrayList(*c.iree_tokenizer_normalizer_t) = try .initCapacity(allocator, sequence.len); + defer normalizers.deinit(allocator); + errdefer for (normalizers.items) |normalizer| c.iree_tokenizer_normalizer_free(normalizer); + + for (sequence) |normalizer| { + normalizers.appendAssumeCapacity(try createNormalizer(normalizer) orelse unreachable); + } + + var normalizer_sequence: ?*c.iree_tokenizer_normalizer_t = null; + // IREE takes ownership of all the normalizers. + try checkOk(c.iree_tokenizer_normalizer_sequence_allocate(normalizers.items.ptr, normalizers.items.len, c.iree_allocator_system(), &normalizer_sequence)); + errdefer c.iree_tokenizer_normalizer_free(normalizer_sequence); + normalizers.clearRetainingCapacity(); + + if (normalizer_sequence == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer_sequence.?)); + if (sequence_size == 0) return error.NormalizerAllocationFailed; + + return .{ + .sequence = normalizer_sequence.?, + .sequence_size = sequence_size, + }; + } + + pub fn deinit(self: *Normalizer) void { + c.iree_tokenizer_normalizer_free(self.sequence); + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + const storage: []align(16) u8 = try allocator.alignedAlloc(u8, .@"16", self.sequence_size); + defer allocator.free(storage); + + var state: ?*c.iree_tokenizer_normalizer_state_t = null; + defer if (state) |ptr| c.iree_tokenizer_normalizer_state_deinitialize(ptr); + + try checkOk(c.iree_tokenizer_normalizer_state_initialize(self.sequence, storage.ptr, &state)); + if (state == null) return error.NormalizationFailed; + + var buffer: [2048]u8 = undefined; + var remaining = text; + + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + + while (remaining.len > 0) { + var written: usize = 0; + var consumed: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_process( + state.?, + .{ .data = remaining.ptr, .size = remaining.len }, + .{ .data = &buffer, .size = buffer.len }, + c.IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END, + &consumed, + &written, + )); + + if (consumed == 0 and written == 0) return error.NormalizationFailed; + + remaining = remaining[consumed..]; + try writer.writer.writeAll(buffer[0..written]); + } + + var has_pending = true; + while (has_pending) { + var written: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_finalize(state.?, .{ .data = &buffer, .size = buffer.len }, &written)); + has_pending = c.iree_tokenizer_normalizer_state_has_pending(state.?); + + try writer.writer.writeAll(buffer[0..written]); + + if (written == 0 and has_pending) return error.NormalizationFailed; + } + + return writer.toOwnedSlice(); + } +}; + +test "normalizer from Hugging Face" { + const allocator = std.testing.allocator; + + const json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ {"type":"NFKD"}, + \\ {"type":"StripAccents"}, + \\ {"type":"Lowercase"}, + \\ {"type":"Replace","pattern":{"Regex":"[ \\t\\r\\n]+"},"content":" "}, + \\ {"type":"Replace","pattern":{"Regex":"^ $"},"content":"\ue000"}, + \\ {"type":"Strip","strip_left":true,"strip_right":true}, + \\ {"type":"Replace","pattern":{"String":"\ue000"},"content":" "} + \\ ] + \\} + ; + + var normalizer = try Normalizer.fromHuggingFaceJson(json); + defer normalizer.deinit(); + + const TestCase = struct { + input: []const u8, + expected: []const u8, + }; + + for ([_]TestCase{ + .{ .input = " hello", .expected = "hello" }, + .{ .input = "hello ", .expected = "hello" }, + .{ .input = " hello ", .expected = "hello" }, + .{ .input = "\nworld\r\n", .expected = "world" }, + .{ .input = "ÉCOLE", .expected = "ecole" }, + .{ .input = " CAFÉ\tAU\nLAIT ", .expected = "cafe au lait" }, + .{ .input = "\t \r\n", .expected = " " }, + .{ .input = "", .expected = "" }, + }) |case| { + const res = try normalizer.normalize(allocator, case.input); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, case.expected, res); + } +} + +test "normalizer sequence" { + const allocator = std.testing.allocator; + + var normalizer = try Normalizer.init(allocator, &.{ + .{ .Strip = .both }, + .Lowercase, + }); + defer normalizer.deinit(); + + const res = try normalizer.normalize(allocator, " helLO WoRLD "); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, "hello world", res); +} + +test "normalizer preserves whitespace across output and sequence boundaries" { + const allocator = std.testing.allocator; + for ([_][]const Normalizer.Kind{ + &.{.{ .Strip = .both }}, + &.{ .Lowercase, .{ .Strip = .both } }, + &.{ .Lowercase, .{ .Strip = .both }, .NFC }, + }) |steps| { + var normalizer = try Normalizer.init(allocator, steps); + defer normalizer.deinit(); + for ([_]usize{ 63, 64, 2047, 2048, 4096 }) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + 4); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..], " b "); + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input[0 .. prefix_len + 2], output); + } + } +} + +test "normalizer sequence preserves Unicode across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.fromHuggingFaceJson( + \\{"type":"Sequence","normalizers":[{"type":"Lowercase"},{"type":"Strip","strip_left":true,"strip_right":true}]} + ); + defer normalizer.deinit(); + for ([_][]const u8{ "é", "€", "😀" }) |codepoint| { + for (61..65) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + codepoint.len + 1); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..][0..codepoint.len], codepoint); + input[input.len - 1] = 'b'; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input, output); + } + } +} + +test "normalizer sequence preserves composition across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); + defer normalizer.deinit(); + const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); +} + +test "normalizer sequence allocation failure retains child ownership" { + const FailAllocator = struct { + fn control(_: ?*anyopaque, _: c.iree_allocator_command_t, _: ?*const anyopaque, _: [*c]?*anyopaque) callconv(.c) c.iree_status_t { + return @ptrFromInt(c.IREE_STATUS_RESOURCE_EXHAUSTED); + } + }; + // Include a nested sequence: failure must preserve its shell and children. + var nested = try Normalizer.init(std.testing.allocator, &.{ .NFC, .Lowercase }); + defer nested.deinit(); + var strip = try Normalizer.init(std.testing.allocator, &.{.{ .Strip = .both }}); + defer strip.deinit(); + const children = [_]*c.iree_tokenizer_normalizer_t{ nested.sequence, strip.sequence }; + var sequence: ?*c.iree_tokenizer_normalizer_t = null; + const status = c.iree_tokenizer_normalizer_sequence_allocate(&children, children.len, .{ + .self = null, + .ctl = FailAllocator.control, + }, &sequence); + defer c.iree_status_free(status); + try std.testing.expect(status != null); + try std.testing.expectEqual(@as(u32, c.IREE_STATUS_RESOURCE_EXHAUSTED), statusCode(status)); + try std.testing.expect(sequence == null); + const output = try nested.normalize(std.testing.allocator, "HELLO"); + defer std.testing.allocator.free(output); + try std.testing.expectEqualStrings("hello", output); +} + +fn tokenOutputWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); +} + +fn writtenTokenIds(out: *std.Io.Writer.Allocating) []const u32 { + const bytes = out.written(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); +} + +fn decodedTextWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.init(allocator); +} + +test "huggingface json encode/decode" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromHuggingFaceJson(allocator, json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const text = "hello world"; + + const token_ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + const token_hello = tokenizer.tokenId("hello") orelse unreachable; + try std.testing.expectEqual(@as(u32, 1), token_hello); + + const unknown = tokenizer.tokenId("missing"); + try std.testing.expect(unknown == null); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +test "loads bpe tokenizer with newline lookahead split and added unk" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "unk_token": "[UNK]", + \\ "vocab": { + \\ "<|UNK|>": 0, + \\ "h": 1, + \\ "e": 2, + \\ "l": 3, + \\ "o": 4, + \\ "he": 5, + \\ "ll": 6, + \\ "lo": 7, + \\ "hel": 8, + \\ "hello": 9 + \\ }, + \\ "merges": ["h e", "l l", "l o", "he l", "hel lo"] + \\ }, + \\ "added_tokens": [ + \\ { + \\ "id": 0, + \\ "content": "<|UNK|>", + \\ "single_word": false, + \\ "lstrip": false, + \\ "rstrip": false, + \\ "normalized": false, + \\ "special": true + \\ } + \\ ], + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "(?:\\r?\\n)+(?!\\r?\\n)"}, + \\ "behavior": "MergedWithNext", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("<|UNK|>").?); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "hello"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 5, 6, 4 }, ids); +} + +const test_mistral_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "vocab": { + \\ "1": 0, "2": 1, "3": 2, "4": 3, "5": 4, "6": 5, "7": 6, + \\ "12": 7, "23": 8, "34": 9, "45": 10, "56": 11, "67": 12, + \\ "123": 13, "234": 14, "345": 15, "456": 16, "567": 17, + \\ "1234": 18 + \\ }, + \\ "merges": ["1 2", "2 3", "3 4", "4 5", "5 6", "6 7", + \\ "12 3", "23 4", "34 5", "45 6", "56 7", "123 4"] + \\ }, + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, + \\ "behavior": "Isolated", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} +; + +test "Mistral right-aligned number lookahead" { + const allocator = std.testing.allocator; + var tokenizer = try Tokenizer.fromBytes(allocator, test_mistral_tokenizer_json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "1", &[_]u32{0} }, + .{ "12", &[_]u32{7} }, + .{ "123", &[_]u32{13} }, + .{ "1234", &[_]u32{ 0, 14 } }, + .{ "12345", &[_]u32{ 7, 15 } }, + .{ "123456", &[_]u32{ 13, 16 } }, + .{ "1234567", &[_]u32{ 0, 14, 17 } }, + }; + inline for (cases) |case| { + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, case[0]); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, case[1], ids); + + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } +} + +test "Mistral Unicode number grouping and output overflow" { + const allocator = std.testing.allocator; + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, test_mistral_tokenizer_json, .{}); + defer parsed.deinit(); + var model = try std.json.parseFromSlice(std.json.Value, allocator, + \\{"type": "WordPiece", "unk_token": "", "continuing_subword_prefix": "", "vocab": { + \\ "": 0, "١": 1, "٢٣٤": 2, "12": 3, "345": 4, + \\ "Ⅷ": 5, "²3४": 6, "1": 7, "111": 8 + \\}} + , .{}); + defer model.deinit(); + const arena = parsed.arena.allocator(); + try parsed.value.object.put(arena, "model", model.value); + // Test the Split segmenter directly, before ByteLevel maps UTF-8 bytes. + const split = parsed.value.object.get("pre_tokenizer").?.object.get("pretokenizers").?.array.items[0]; + try parsed.value.object.put(arena, "pre_tokenizer", split); + const json = try std.json.Stringify.valueAlloc(allocator, parsed.value, .{}); + defer allocator.free(json); + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "١٢٣٤", &[_]u32{ 1, 2 } }, + .{ "12345", &[_]u32{ 3, 4 } }, + .{ "Ⅷ²3४", &[_]u32{ 5, 6 } }, + }; + inline for (cases) |case| { + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } + + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, &(@as([1000]u8, @splat('1')))); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 334), ids.len); + try std.testing.expectEqual(@as(u32, 7), ids[0]); + for (ids[1..]) |id| try std.testing.expectEqual(@as(u32, 8), id); +} + +test "writer" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + const text = "hello world"; + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const token_ids_alloc = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids_alloc); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc); + + encoder.reset(); + var buf: [1024]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + const token_ids = writtenTokenIds(&out); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +/// Shared tokenizer JSON config used by multiple tests. +const test_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2, + \\ "foo": 3, + \\ "bar": 4, + \\ "a": 5, + \\ "b": 6, + \\ "c": 7 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} +; + +test "writer with tiny buffer forces drain" { + // A buffer smaller than the input forces the drain path (not just flush). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // Buffer of 4 bytes; "hello world" is 11 bytes, so drain must be invoked. + var buf: [4]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "writer byte-by-byte" { + // Feed one byte at a time through the writer to stress buffer/drain boundaries. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var buf: [1]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + for ("hello world") |byte| { + try writer.interface.writeByte(byte); + } + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "encoder reset between multiple encodes" { + // Ensure reset properly clears state so successive encodes are independent. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + { + const ids = try encoder.encodeAlloc(allocator, "foo bar"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 3, 4 }, ids); + } + + encoder.reset(); + + { + const ids = try encoder.encodeAlloc(allocator, "hello world"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, ids); + } + + encoder.reset(); + + // Third encode to verify repeated resets work. + { + const ids = try encoder.encodeAlloc(allocator, "foo"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{3}, ids); + } +} + +test "writer and encodeAlloc produce same results" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + "foo bar hello world a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const expected = try encoder.encodeAlloc(allocator, text); + defer allocator.free(expected); + + encoder.reset(); + + var buf: [8]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, expected, writtenTokenIds(&out)); + } +} + +test "encode empty string" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // encodeAlloc with empty string should produce no tokens. + { + const ids = try encoder.encodeAlloc(allocator, ""); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 0), ids.len); + } + + encoder.reset(); + + // Writer with empty string should also produce no tokens. + { + var buf: [16]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(""); + try writer.finalize(); + try std.testing.expectEqual(@as(usize, 0), writtenTokenIds(&out).len); + } +} + +test "unknown tokens" { + // Words not in the vocabulary should map to the [UNK] token (id 0). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "unknown"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{0}, ids); +} + +test "decode round-trip" { + // Encode then decode several strings and verify round-trip fidelity. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(ids, &decoded.writer); + + try std.testing.expectEqualStrings(text, decoded.written()); + } +} + +test "decoder reset between decodes" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 1, 2 }, &text.writer); + try std.testing.expectEqualStrings("hello world", text.written()); + } + + try decoder.reset(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 3, 4 }, &text.writer); + try std.testing.expectEqualStrings("foo bar", text.written()); + } +} + +test "tokenId lookup" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("[UNK]").?); + try std.testing.expectEqual(@as(u32, 1), tokenizer.tokenId("hello").?); + try std.testing.expectEqual(@as(u32, 2), tokenizer.tokenId("world").?); + try std.testing.expectEqual(@as(u32, 3), tokenizer.tokenId("foo").?); + try std.testing.expectEqual(@as(u32, 4), tokenizer.tokenId("bar").?); + try std.testing.expect(tokenizer.tokenId("nonexistent") == null); + try std.testing.expect(tokenizer.tokenId("") == null); +} + +test "tokenId <-> decode" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + for ([_][]const u8{ "[UNK]", "hello", "world", "foo", "bar" }) |text| { + try std.testing.expectEqualSlices(u8, text, tokenizer.decode(tokenizer.tokenId(text).?).?); + } +} + +test "writer with pre-allocated output writer" { + // Pass an already-allocated aligned output writer to the encoder stream. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var out = tokenOutputWriter(allocator); + defer out.deinit(); + try out.ensureUnusedCapacity(64 * @sizeOf(u32)); + + var buf: [32]u8 = undefined; + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index a35de7d5..14eb1a87 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -12,16 +12,15 @@ const Flags = struct { verbose: bool = false, }; -pub fn main() !void { - var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; - const allocator = gpa.allocator(); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); const io = threaded.io(); - const args = stdx.flags.parseProcessArgs(Flags); + const args = stdx.flags.parseProcessArgs(init.minimal, Flags); log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); @@ -34,15 +33,17 @@ pub fn main() !void { var decoder = try tokenizer.decoder(); defer decoder.deinit(); - const prompt_tok = try encoder.encode(args.prompt); + const prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); + defer allocator.free(prompt_tok); log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); var errors: u8 = 0; { - const reconstructed = try decoder.decode(prompt_tok); - if (!std.mem.eql(u8, args.prompt, reconstructed)) { - log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); + var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok); + defer reconstructed.deinit(allocator); + if (!std.mem.eql(u8, args.prompt, reconstructed.items)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed.items}); errors += 1; } } diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index 0fe8e061..dd4d2e76 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_zig//zig:defs.bzl", "zig_library") swig_cc_library( name = "sentencepiece_swig", + tags = ["manual"], interface = "sentencepiece.i", module = "sentencepiece", deps = [ @@ -15,6 +16,7 @@ swig_cc_library( zig_library( name = "sentencepiece", + tags = ["manual"], import_name = "sentencepiece", main = "sentencepiece.zig", visibility = ["//visibility:public"], @@ -25,12 +27,8 @@ zig_library( ], ) -zig_srcs( - name = "sources", - zig_lib = ":sentencepiece", -) - -cc_static_library( - name="sentencepiece_static", - deps = [":sentencepiece_swig"] -) +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":sentencepiece", +# ) diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i index 35ccc057..d89489c7 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.i +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -19,7 +19,7 @@ $1 = absl::string_view((char *)$input.ptr, $input.len); } -%typemap(out, optimal="1") const std::string& %{ +%typemap(out) const std::string& %{ $result.ptr = (void *)($1->data()); $result.len = (size_t)($1->length()); %} @@ -85,6 +85,8 @@ namespace sentencepiece { virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; virtual int PieceToId(absl::string_view piece) const; + virtual const std::string &IdToPiece(int id) const; + virtual int GetPieceSize() const; virtual int unk_id() const; virtual int bos_id() const; virtual int eos_id() const; diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index c98b0740..a09e0c0a 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -89,7 +89,7 @@ pub const Decoder = struct { inner: *SentencePieceProcessor, vec: *c.std_vector_int, str: *c.std_string, - last_string: StringBuffer = .{ .len = 0 }, + last_string: StringBuffer = .empty, fn init(inner: *SentencePieceProcessor) !Decoder { const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); @@ -192,7 +192,12 @@ pub const SentencePieceProcessor = opaque { return try Decoder.init(self); } - pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { + pub fn tokenId(self: *SentencePieceProcessor, token: []const u8) u32 { return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); } + + pub fn decode(self: *SentencePieceProcessor, token_id: u32) ?[]const u8 { + if (token_id >= @as(u32, @intCast(c.SentencePieceProcessor_GetPieceSize(@ptrCast(self))))) return null; + return ffi.ZigSlice.to(u8, c.SentencePieceProcessor_IdToPiece(@ptrCast(self), @intCast(token_id))); + } }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index c72ec803..fc304b81 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,25 +1,40 @@ const std = @import("std"); -pub const hftokenizers = @import("hftokenizers"); +pub const iree = @import("iree"); pub const sentencepiece = @import("sentencepiece"); -pub const homemade = @import("homemade.zig"); - -test { - std.testing.refAllDecls(@This()); -} +const log = std.log.scoped(.@"zml/tokenizer"); const Tokenizers = enum { - hftokenizers, + iree, sentencepiece, - homemade, }; pub const Tokenizer = union(Tokenizers) { + pub const Normalizer = union(Tokenizers) { + pub const Kind = iree.Normalizer.Kind; + + iree: iree.Normalizer, + sentencepiece: void, + + pub fn deinit(self: *Normalizer) void { + switch (self.*) { + .iree => |*n| n.deinit(), + else => {}, + } + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + return switch (self.*) { + .iree => |*n| n.normalize(allocator, text), + else => error.UnsupportedNormalizer, + }; + } + }; + pub const Encoder = union(Tokenizers) { - hftokenizers: hftokenizers.Encoder, + iree: iree.Tokenizer.Encoder, sentencepiece: sentencepiece.Encoder, - homemade: homemade.Encoder, pub fn deinit(self: *Encoder) void { switch (self.*) { @@ -33,23 +48,34 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { return switch (self.*) { - inline else => |*v| v.encode(input), + .iree => |*encoder_| encoder_.encodeAlloc(allocator, text), + inline else => |*encoder_| try allocator.dupe(u32, try encoder_.encode(text)), }; } - pub fn ids(self: Encoder) []const u32 { - return switch (self) { - inline else => |v| v.ids(), + pub fn encode(self: *Encoder, writer: *std.Io.Writer, text: []const u8) !void { + return switch (self.*) { + .iree => |*e| { + try e.feed(text, writer); + try e.finalize(writer); + }, + inline else => |*e| { + const encoded: []const u32 = try e.encode(text); + try writer.writeAll(std.mem.sliceAsBytes(encoded)); + }, }; } + + pub fn appendTokens(_: *Encoder, writer: *std.Io.Writer, tokens: []const u32) !void { + try writer.writeAll(@ptrCast(tokens)); + } }; pub const Decoder = union(Tokenizers) { - hftokenizers: hftokenizers.Decoder, + iree: iree.Tokenizer.Decoder, sentencepiece: sentencepiece.Decoder, - homemade: homemade.Decoder, pub fn deinit(self: *Decoder) void { switch (self.*) { @@ -63,81 +89,101 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { + pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { return switch (self.*) { - inline else => |*v| v.decode(ids_), + .iree => |*d| { + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + try d.decode(token_ids, &writer.writer); + return writer.toArrayList(); + }, + inline else => |*d| .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))), }; } - pub fn string(self: *Decoder) []const u8 { + // Only valid until the next decode and while the decoder AND buffer is alive. + pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { return switch (self.*) { - inline else => |v| v.string(), + .iree => |*d| d.feedOne(token_id, buffer), + inline else => |*d| try d.next(token_id) orelse &.{}, }; } - //pub fn ids(self: *Decoder) []u32 { - // return switch (self.*) { - // inline else => |v| v.ids(), - // }; - //} - - pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. + pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { return switch (self.*) { - inline else => |*v| v.next(token_id), + .iree => |*v| v.finalize(buffer), + inline else => &.{}, }; } }; - hftokenizers: *hftokenizers.HFTokenizer, + iree: iree.Tokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, - homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .hftokenizers = try .fromFile(model) }; + return .{ .iree = try .fromFile(allocator, io, model) }; } return error.InvalidArgument; } - pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { - _ = allocator; // autofix - _ = io; // autofix + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { if (bytes[0] == '{') { - return .{ .hftokenizers = try .fromBytes(bytes) }; + return .{ .iree = try .fromBytes(allocator, bytes) }; } return .{ .sentencepiece = try .fromBytes(bytes) }; } pub fn deinit(self: *Tokenizer) void { switch (self.*) { - inline else => |t| t.deinit(), + inline else => |*t| t.*.deinit(), } } - pub fn encoder(self: Tokenizer) !Encoder { - return switch (self) { - inline else => |v, tag| @unionInit(Encoder, @tagName(tag), try v.encoder()), + pub fn encoder(self: *const Tokenizer) !Encoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Encoder, @tagName(tag), try v.*.encoder()), }; } - pub fn decoder(self: Tokenizer) !Decoder { - return switch (self) { - inline else => |v, tag| @unionInit(Decoder, @tagName(tag), try v.decoder()), + pub fn decoder(self: *const Tokenizer) !Decoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Decoder, @tagName(tag), try v.*.decoder()), }; } - pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { - return switch (self) { - inline else => |v| v.tokenToId(token), + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizer(sequence) }, + else => error.UnsupportedNormalizer, + }; + } + + pub fn normalizerFromHuggingFaceJson(self: *const Tokenizer, json: []const u8) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizerFromHuggingFaceJson(json) }, + else => error.UnsupportedNormalizer, + }; + } + + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + return switch (self.*) { + inline else => |v| v.tokenId(token), + }; + } + + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + return switch (self.*) { + inline else => |v| v.decode(token_id), }; } }; -test {} -test {} +test { + std.testing.refAllDecls(@This()); +} diff --git a/zml/tracer.zig b/zml/tracer.zig deleted file mode 100644 index e2e173ef..00000000 --- a/zml/tracer.zig +++ /dev/null @@ -1,77 +0,0 @@ -const builtin = @import("builtin"); - -const c = @import("c"); - -pub const Tracer = switch (builtin.os.tag) { - // TODO(cerisier): fix MacOsTracer - // .macos => MacOsTracer, - .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, - else => FakeTracer, -}; - -const CudaTracer = struct { - - // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so - // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). - const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; - const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; - - // Those symbols are defined in nvToolsExt.h which we don't want to provide. - // However, we link with libnvToolsExt.so which provides them. - // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). - const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; - const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; - const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; - - pub fn event(message: [:0]const u8) void { - nvtxMarkA(message.ptr); - } - - pub fn frameStart(message: [:0]const u8) u64 { - return @intCast(nvtxRangeStartA(message.ptr)); - } - - pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { - _ = message; - nvtxRangeEnd(@intCast(interval_id)); - return; - } -}; - -const MacOsTracer = struct { - logger: c.os_log_t, - - pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_event(self.logger, interval_id, message); - } - - pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_interval_begin(self.logger, interval_id, message); - return interval_id; - } - - pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { - c.zml_os_signpost_interval_end(self.logger, interval_id, message); - } -}; - -/// Mock tracer for OS which don't have an impl. -const FakeTracer = struct { - pub fn event(message: [:0]const u8) void { - _ = message; - return; - } - - pub fn frameStart(message: [:0]const u8) u64 { - _ = message; - return 0; - } - - pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { - _ = interval_id; - _ = message; - return; - } -}; diff --git a/zml/zml.zig b/zml/zml.zig index 22faac3c..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -13,37 +13,65 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention.zig"); +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; +pub const Compiler = @import("Compiler.zig"); pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; +pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); +pub const kernel = @import("kernel/kernel.zig"); +pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); -pub const module = @import("module.zig"); +pub const moe = @import("kernels/moe/moe.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); +pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; -pub const CompilationOptions = platform.CompilationOptions; +pub const tracer = @import("profiling/tracer.zig"); +pub const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; +pub const Sharding = @import("Sharding.zig"); +pub const Mesh = Sharding.Mesh; pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); -pub const tracer = @import("tracer.zig"); + +/// Deprecated, use `zml.Compiler` instead +pub const module = struct { + pub const CompilationContext = Compiler; + pub const CompileError = Compiler.Error; + pub const CompilationOptions = Compiler.Options; + + /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` + pub fn compile( + allocator: std.mem.Allocator, + io_: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform_: *const Platform, + opts: Compiler.Options, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io_, platform_, func, args, opts); + } +}; test "zml" { std.testing.refAllDecls(@This()); @@ -54,6 +82,8 @@ pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; -pub fn init(init_: std.process.Init) !void { - try bazel.initSimple(init_); +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); }